[llvm] [Hexagon] Support partial reduction intrinsics (PR #179797)
Alexey Karyakin via llvm-commits
llvm-commits at lists.llvm.org
Fri Feb 13 07:34:46 PST 2026
https://github.com/quic-akaryaki updated https://github.com/llvm/llvm-project/pull/179797
>From 441261a38c33802f1c9997604da567cfd0b44bbb Mon Sep 17 00:00:00 2001
From: Alexey Karyakin <akaryaki at qti.qualcomm.com>
Date: Wed, 4 Feb 2026 13:45:55 -0800
Subject: [PATCH 1/6] [Hexagon] Support partial reduction intrinsics
@llvm.vector.partial.reduce.add.* intrinsics are translated to
PARTIAL_REDUCE_(U|S|SU)MLA. On Hexagon, these are lowered to
accumulating vrmpy.
---
.../Target/Hexagon/HexagonISelLoweringHVX.cpp | 15 +++
llvm/lib/Target/Hexagon/HexagonPatternsHVX.td | 7 ++
.../CodeGen/Hexagon/hvx-partial-reduce.ll | 108 ++++++++++++++++++
3 files changed, 130 insertions(+)
create mode 100644 llvm/test/CodeGen/Hexagon/hvx-partial-reduce.ll
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
index 363f9fd69e3b9..e427ce55e25ff 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
@@ -498,6 +498,21 @@ HexagonTargetLowering::initializeHVXLowering() {
setOperationAction(ISD::SINT_TO_FP, MVT::v32i1, Custom);
setTargetDAGCombine({ISD::CONCAT_VECTORS, ISD::TRUNCATE, ISD::VSELECT});
+
+ // Partial reductions.
+ {
+ static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA,
+ ISD::PARTIAL_REDUCE_UMLA,
+ ISD::PARTIAL_REDUCE_SUMLA};
+
+ auto HvxType = [=](MVT ScalarT) {
+ return MVT::getVectorVT(ScalarT, Subtarget.getVectorLength() * 8 /
+ ScalarT.getSizeInBits());
+ };
+
+ setPartialReduceMLAAction(MLAOps, HvxType(MVT::i32), HvxType(MVT::i8),
+ Legal);
+ }
}
unsigned
diff --git a/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td b/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td
index c6ea6d02bb5d8..a4b8d895672a0 100644
--- a/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td
+++ b/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td
@@ -456,6 +456,13 @@ let Predicates = [UseHVX] in {
(VShuff (V6_vmpyhus_acc (VDeal $Vx, -4),
HVI16:$Vs, HVI16:$Vt), -4)>;
}
+
+ def : Pat<(VecI32 (partial_reduce_umla VecI32:$Acc, HVI8:$A, HVI8:$B)),
+ (V6_vrmpyubv_acc $Acc, $A, $B)>;
+ def : Pat<(VecI32 (partial_reduce_smla VecI32:$Acc, HVI8:$A, HVI8:$B)),
+ (V6_vrmpybv_acc $Acc, $A, $B)>;
+ def : Pat<(VecI32 (partial_reduce_sumla VecI32:$Acc, HVI8:$A, HVI8:$B)),
+ (V6_vrmpybusv_acc $Acc, $B, $A)>;
}
let Predicates = [UseHVX] in {
diff --git a/llvm/test/CodeGen/Hexagon/hvx-partial-reduce.ll b/llvm/test/CodeGen/Hexagon/hvx-partial-reduce.ll
new file mode 100644
index 0000000000000..0165d5933a7d1
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/hvx-partial-reduce.ll
@@ -0,0 +1,108 @@
+;; Check HVX vectorization.
+; RUN: llc -mtriple hexagon < %s | FileCheck %s --check-prefixes=CHECK,CHECK-HVX
+
+;; Check that there is no failure when compiling to scalar code, don't check the output.
+; RUN: llc -mtriple hexagon -mattr=-hvx,-hvxv73,-hvx-length128b < %s | FileCheck %s --check-prefixes=CHECK,CHECK-NO-HVX
+
+define <16 x i32> @partial_reduce_uu_64(<16 x i32> %acc, <64 x i8> %x, <64 x i8> %y) #0 {
+; CHECK-LABEL: partial_reduce_uu_64:
+; CHECK-HVX: v0.uw += vrmpy(v1.ub,v2.ub)
+; CHECK-NO-HVX: {{r[0-9]+}} += mpyi
+ %x.ext = zext <64 x i8> %x to <64 x i32>
+ %y.ext = zext <64 x i8> %y to <64 x i32>
+ %m = mul nuw nsw <64 x i32> %x.ext, %y.ext
+ %partial.reduce = tail call <16 x i32> @llvm.vector.partial.reduce.add.v32i32.v128i32(<16 x i32> %acc, <64 x i32> %m)
+ ret <16 x i32> %partial.reduce
+}
+
+define <16 x i32> @partial_reduce_su_64(<16 x i32> %acc, <64 x i8> %x, <64 x i8> %y) #0 {
+; CHECK-LABEL: partial_reduce_su_64:
+; CHECK-HVX: v0.w += vrmpy(v2.ub,v1.b)
+; CHECK-NO-HVX: {{r[0-9]+}} += mpyi
+ %x.ext = sext <64 x i8> %x to <64 x i32>
+ %y.ext = zext <64 x i8> %y to <64 x i32>
+ %m = mul nuw nsw <64 x i32> %x.ext, %y.ext
+ %partial.reduce = tail call <16 x i32> @llvm.vector.partial.reduce.add.v32i32.v128i32(<16 x i32> %acc, <64 x i32> %m)
+ ret <16 x i32> %partial.reduce
+}
+
+define <16 x i32> @partial_reduce_us_64(<16 x i32> %acc, <64 x i8> %x, <64 x i8> %y) #0 {
+; CHECK-LABEL: partial_reduce_us_64:
+; CHECK-HVX: v0.w += vrmpy(v1.ub,v2.b)
+; CHECK-NO-HVX: {{r[0-9]+}} += mpyi
+ %x.ext = zext <64 x i8> %x to <64 x i32>
+ %y.ext = sext <64 x i8> %y to <64 x i32>
+ %m = mul nuw nsw <64 x i32> %x.ext, %y.ext
+ %partial.reduce = tail call <16 x i32> @llvm.vector.partial.reduce.add.v32i32.v128i32(<16 x i32> %acc, <64 x i32> %m)
+ ret <16 x i32> %partial.reduce
+}
+
+define <16 x i32> @partial_reduce_ss_64(<16 x i32> %acc, <64 x i8> %x, <64 x i8> %y) #0 {
+; CHECK-LABEL: partial_reduce_ss_64:
+; CHECK-HVX: v0.w += vrmpy(v1.b,v2.b)
+; CHECK-NO-HVX: {{r[0-9]+}} += mpyi
+ %x.ext = sext <64 x i8> %x to <64 x i32>
+ %y.ext = sext <64 x i8> %y to <64 x i32>
+ %m = mul nuw nsw <64 x i32> %x.ext, %y.ext
+ %partial.reduce = tail call <16 x i32> @llvm.vector.partial.reduce.add.v32i32.v128i32(<16 x i32> %acc, <64 x i32> %m)
+ ret <16 x i32> %partial.reduce
+}
+
+define <32 x i32> @partial_reduce_uu_128(<32 x i32> %acc, <128 x i8> %x, <128 x i8> %y) #1 {
+; CHECK-LABEL: partial_reduce_uu_128:
+; CHECK-HVX: v0.uw += vrmpy(v1.ub,v2.ub)
+; CHECK-NO-HVX: {{r[0-9]+}} += mpyi
+ %x.ext = zext <128 x i8> %x to <128 x i32>
+ %y.ext = zext <128 x i8> %y to <128 x i32>
+ %m = mul nuw nsw <128 x i32> %x.ext, %y.ext
+ %partial.reduce = tail call <32 x i32> @llvm.vector.partial.reduce.add.v32i32.v128i32(<32 x i32> %acc, <128 x i32> %m)
+ ret <32 x i32> %partial.reduce
+}
+
+define <32 x i32> @partial_reduce_su_128(<32 x i32> %acc, <128 x i8> %x, <128 x i8> %y) #1 {
+; CHECK-LABEL: partial_reduce_su_128:
+; CHECK-HVX: v0.w += vrmpy(v2.ub,v1.b)
+; CHECK-NO-HVX: {{r[0-9]+}} += mpyi
+ %x.ext = sext <128 x i8> %x to <128 x i32>
+ %y.ext = zext <128 x i8> %y to <128 x i32>
+ %m = mul nuw nsw <128 x i32> %x.ext, %y.ext
+ %partial.reduce = tail call <32 x i32> @llvm.vector.partial.reduce.add.v32i32.v128i32(<32 x i32> %acc, <128 x i32> %m)
+ ret <32 x i32> %partial.reduce
+}
+
+define <32 x i32> @partial_reduce_us_128(<32 x i32> %acc, <128 x i8> %x, <128 x i8> %y) #1 {
+; CHECK-LABEL: partial_reduce_us_128:
+; CHECK-HVX: v0.w += vrmpy(v1.ub,v2.b)
+; CHECK-NO-HVX: {{r[0-9]+}} += mpyi
+ %x.ext = zext <128 x i8> %x to <128 x i32>
+ %y.ext = sext <128 x i8> %y to <128 x i32>
+ %m = mul nuw nsw <128 x i32> %x.ext, %y.ext
+ %partial.reduce = tail call <32 x i32> @llvm.vector.partial.reduce.add.v32i32.v128i32(<32 x i32> %acc, <128 x i32> %m)
+ ret <32 x i32> %partial.reduce
+}
+
+define <32 x i32> @partial_reduce_ss_128(<32 x i32> %acc, <128 x i8> %x, <128 x i8> %y) #1 {
+; CHECK-LABEL: partial_reduce_ss_128:
+; CHECK-HVX: v0.w += vrmpy(v1.b,v2.b)
+; CHECK-NO-HVX: {{r[0-9]+}} += mpyi
+ %x.ext = sext <128 x i8> %x to <128 x i32>
+ %y.ext = sext <128 x i8> %y to <128 x i32>
+ %m = mul nuw nsw <128 x i32> %x.ext, %y.ext
+ %partial.reduce = tail call <32 x i32> @llvm.vector.partial.reduce.add.v32i32.v128i32(<32 x i32> %acc, <128 x i32> %m)
+ ret <32 x i32> %partial.reduce
+}
+
+;; Check for vector size that do not match an available vrmpy (2x reduction).
+define <64 x i32> @partial_reduce_unsupported(<64 x i32> %acc, <128 x i8> %x, <128 x i8> %y) #1 {
+; CHECK-LABEL: partial_reduce_unsupported:
+; CHECK-HVX: vmpy
+; CHECK-HVX: vadd
+ %x.ext = zext <128 x i8> %x to <128 x i32>
+ %y.ext = zext <128 x i8> %y to <128 x i32>
+ %m = mul nuw nsw <128 x i32> %x.ext, %y.ext
+ %partial.reduce = tail call <64 x i32> @llvm.vector.partial.reduce.add.v64i32.v128i32(<64 x i32> %acc, <128 x i32> %m)
+ ret <64 x i32> %partial.reduce
+}
+
+attributes #0 = { nounwind "target-cpu"="hexagonv73" "target-features"="+hvxv73,+hvx-length64b" }
+attributes #1 = { nounwind "target-cpu"="hexagonv73" "target-features"="+hvxv73,+hvx-length128b" }
>From cab260353cd04a105bce449b3ab7e5b0c2cd020e Mon Sep 17 00:00:00 2001
From: Alexey Karyakin <akaryaki at qti.qualcomm.com>
Date: Wed, 4 Feb 2026 08:28:44 -0800
Subject: [PATCH 2/6] [Hexagon] NFC: Convert PerformDAGCombine to switch
Replace a chained if-else to a switch, as it is in the downstream code.
---
llvm/lib/Target/Hexagon/HexagonISelLowering.cpp | 17 +++++++++++++----
1 file changed, 13 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 0ceb095cdc653..3d9b34549b214 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -3431,7 +3431,8 @@ HexagonTargetLowering::PerformDAGCombine(SDNode *N,
if (DCI.isBeforeLegalizeOps())
return SDValue();
- if (Opc == HexagonISD::P2D) {
+ switch (Opc) {
+ case HexagonISD::P2D: {
SDValue P = Op.getOperand(0);
switch (P.getOpcode()) {
case HexagonISD::PTRUE:
@@ -3441,7 +3442,9 @@ HexagonTargetLowering::PerformDAGCombine(SDNode *N,
default:
break;
}
- } else if (Opc == ISD::VSELECT) {
+ break;
+ }
+ case ISD::VSELECT: {
// This is pretty much duplicated in HexagonISelLoweringHVX...
//
// (vselect (xor x, ptrue), v0, v1) -> (vselect x, v1, v0)
@@ -3454,7 +3457,9 @@ HexagonTargetLowering::PerformDAGCombine(SDNode *N,
return VSel;
}
}
- } else if (Opc == ISD::TRUNCATE) {
+ break;
+ }
+ case ISD::TRUNCATE: {
SDValue Op0 = Op.getOperand(0);
// fold (truncate (build pair x, y)) -> (truncate x) or x
if (Op0.getOpcode() == ISD::BUILD_PAIR) {
@@ -3467,7 +3472,9 @@ HexagonTargetLowering::PerformDAGCombine(SDNode *N,
if (ty(Elem0).bitsGT(TruncTy))
return DCI.DAG.getNode(ISD::TRUNCATE, dl, TruncTy, Elem0);
}
- } else if (Opc == ISD::OR) {
+ break;
+ }
+ case ISD::OR: {
// fold (or (shl xx, s), (zext y)) -> (COMBINE (shl xx, s-32), y)
// if s >= 32
auto fold0 = [&, this](SDValue Op) {
@@ -3497,6 +3504,8 @@ HexagonTargetLowering::PerformDAGCombine(SDNode *N,
if (SDValue R = fold0(Op))
return R;
+ break;
+ }
}
return SDValue();
>From 9857e2f655f1c7d4dcb6001d124c69fe05e9f214 Mon Sep 17 00:00:00 2001
From: Alexey Karyakin <akaryaki at qti.qualcomm.com>
Date: Wed, 4 Feb 2026 10:43:41 -0800
Subject: [PATCH 3/6] [Hexagon] NFC: Add common case for DAG combines
HexagonTargetLowering::PerformDAGCombine() had two mutually-exclusive
paths: 1) for operations on vectors that match HVX registers or can be
widened and 2) for everything else. However, some transformations are
applicable to arbitrary vector sizes. This lead to duplication of
VSELECT combining. To remove this duplication and allow for easier
addition of more common operations, create a common code path.
---
.../Target/Hexagon/HexagonISelLowering.cpp | 42 +++++++------
.../Target/Hexagon/HexagonISelLoweringHVX.cpp | 60 ++++++++-----------
2 files changed, 48 insertions(+), 54 deletions(-)
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 3d9b34549b214..6f1e52998996d 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -3403,16 +3403,35 @@ HexagonTargetLowering::ReplaceNodeResults(SDNode *N,
SDValue
HexagonTargetLowering::PerformDAGCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
+ SDValue Op(N, 0);
+ const SDLoc &dl(Op);
+ unsigned Opc = Op.getOpcode();
+
+ // Combining transformations applicable for arbitrary vector sizes.
+ if (!DCI.isBeforeLegalizeOps()) {
+ switch (Opc) {
+ case ISD::VSELECT: {
+ // (vselect (xor x, ptrue), v0, v1) -> (vselect x, v1, v0)
+ SDValue Cond = Op.getOperand(0);
+ if (Cond->getOpcode() == ISD::XOR) {
+ SDValue C0 = Cond.getOperand(0), C1 = Cond.getOperand(1);
+ if (C1->getOpcode() == HexagonISD::PTRUE) {
+ SDValue VSel = DCI.DAG.getNode(ISD::VSELECT, dl, ty(Op), C0,
+ Op.getOperand(2), Op.getOperand(1));
+ return VSel;
+ }
+ }
+ return SDValue();
+ }
+ }
+ }
+
if (isHvxOperation(N, DCI.DAG)) {
if (SDValue V = PerformHvxDAGCombine(N, DCI))
return V;
return SDValue();
}
- SDValue Op(N, 0);
- const SDLoc &dl(Op);
- unsigned Opc = Op.getOpcode();
-
if (Opc == ISD::TRUNCATE) {
SDValue Op0 = Op.getOperand(0);
// fold (truncate (build pair x, y)) -> (truncate x) or x
@@ -3444,21 +3463,6 @@ HexagonTargetLowering::PerformDAGCombine(SDNode *N,
}
break;
}
- case ISD::VSELECT: {
- // This is pretty much duplicated in HexagonISelLoweringHVX...
- //
- // (vselect (xor x, ptrue), v0, v1) -> (vselect x, v1, v0)
- SDValue Cond = Op.getOperand(0);
- if (Cond->getOpcode() == ISD::XOR) {
- SDValue C0 = Cond.getOperand(0), C1 = Cond.getOperand(1);
- if (C1->getOpcode() == HexagonISD::PTRUE) {
- SDValue VSel = DCI.DAG.getNode(ISD::VSELECT, dl, ty(Op), C0,
- Op.getOperand(2), Op.getOperand(1));
- return VSel;
- }
- }
- break;
- }
case ISD::TRUNCATE: {
SDValue Op0 = Op.getOperand(0);
// fold (truncate (build pair x, y)) -> (truncate x) or x
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
index e427ce55e25ff..bebff0ca4e7bd 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
@@ -3924,43 +3924,33 @@ HexagonTargetLowering::PerformHvxDAGCombine(SDNode *N, DAGCombinerInfo &DCI)
return SDValue();
switch (Opc) {
- case ISD::VSELECT: {
- // (vselect (xor x, qtrue), v0, v1) -> (vselect x, v1, v0)
- SDValue Cond = Ops[0];
- if (Cond->getOpcode() == ISD::XOR) {
- SDValue C0 = Cond.getOperand(0), C1 = Cond.getOperand(1);
- if (C1->getOpcode() == HexagonISD::QTRUE)
- return DAG.getNode(ISD::VSELECT, dl, ty(Op), C0, Ops[2], Ops[1]);
- }
- break;
+ case HexagonISD::V2Q:
+ if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR) {
+ if (const auto *C = dyn_cast<ConstantSDNode>(Ops[0].getOperand(0)))
+ return C->isZero() ? DAG.getNode(HexagonISD::QFALSE, dl, ty(Op))
+ : DAG.getNode(HexagonISD::QTRUE, dl, ty(Op));
}
- case HexagonISD::V2Q:
- if (Ops[0].getOpcode() == ISD::SPLAT_VECTOR) {
- if (const auto *C = dyn_cast<ConstantSDNode>(Ops[0].getOperand(0)))
- return C->isZero() ? DAG.getNode(HexagonISD::QFALSE, dl, ty(Op))
- : DAG.getNode(HexagonISD::QTRUE, dl, ty(Op));
- }
- break;
- case HexagonISD::Q2V:
- if (Ops[0].getOpcode() == HexagonISD::QTRUE)
- return DAG.getNode(ISD::SPLAT_VECTOR, dl, ty(Op),
- DAG.getAllOnesConstant(dl, MVT::i32));
- if (Ops[0].getOpcode() == HexagonISD::QFALSE)
- return getZero(dl, ty(Op), DAG);
- break;
- case HexagonISD::VINSERTW0:
- if (isUndef(Ops[1]))
- return Ops[0];
- break;
- case HexagonISD::VROR: {
- if (Ops[0].getOpcode() == HexagonISD::VROR) {
- SDValue Vec = Ops[0].getOperand(0);
- SDValue Rot0 = Ops[1], Rot1 = Ops[0].getOperand(1);
- SDValue Rot = DAG.getNode(ISD::ADD, dl, ty(Rot0), {Rot0, Rot1});
- return DAG.getNode(HexagonISD::VROR, dl, ty(Op), {Vec, Rot});
- }
- break;
+ break;
+ case HexagonISD::Q2V:
+ if (Ops[0].getOpcode() == HexagonISD::QTRUE)
+ return DAG.getNode(ISD::SPLAT_VECTOR, dl, ty(Op),
+ DAG.getAllOnesConstant(dl, MVT::i32));
+ if (Ops[0].getOpcode() == HexagonISD::QFALSE)
+ return getZero(dl, ty(Op), DAG);
+ break;
+ case HexagonISD::VINSERTW0:
+ if (isUndef(Ops[1]))
+ return Ops[0];
+ break;
+ case HexagonISD::VROR: {
+ if (Ops[0].getOpcode() == HexagonISD::VROR) {
+ SDValue Vec = Ops[0].getOperand(0);
+ SDValue Rot0 = Ops[1], Rot1 = Ops[0].getOperand(1);
+ SDValue Rot = DAG.getNode(ISD::ADD, dl, ty(Rot0), {Rot0, Rot1});
+ return DAG.getNode(HexagonISD::VROR, dl, ty(Op), {Vec, Rot});
}
+ break;
+ }
}
return SDValue();
>From 0bb0ddd6cc204329643b028ffe7c86585817c784 Mon Sep 17 00:00:00 2001
From: Alexey Karyakin <akaryaki at qti.qualcomm.com>
Date: Wed, 4 Feb 2026 14:08:33 -0800
Subject: [PATCH 4/6] [Hexagon] NFC: Add test for vecreduce_add
---
.../CodeGen/Hexagon/expand-vecreduce-add.ll | 143 ++++++++++++++++++
1 file changed, 143 insertions(+)
create mode 100644 llvm/test/CodeGen/Hexagon/expand-vecreduce-add.ll
diff --git a/llvm/test/CodeGen/Hexagon/expand-vecreduce-add.ll b/llvm/test/CodeGen/Hexagon/expand-vecreduce-add.ll
new file mode 100644
index 0000000000000..cd64099faebb4
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/expand-vecreduce-add.ll
@@ -0,0 +1,143 @@
+; RUN: llc -mtriple=hexagon < %s | FileCheck %s
+
+target triple = "hexagon"
+
+define i32 @add_v32i32(<32 x i32> %vec) #0 {
+; CHECK-LABEL: add_v32i32:
+; CHECK: {
+; CHECK: [[R0:v[0-9]+]] = valign([[_:v[0-9]+]],v0,{{.+}})
+; CHECK: }
+; CHECK: {
+; CHECK: [[R1:v[0-9]+]].w = vadd(v0.w,[[R0]].w)
+; CHECK: }
+; CHECK: {
+; CHECK: [[R2:v[0-9]+]] = valign([[_:v[0-9]+]],[[R1]],{{.+}})
+; CHECK: }
+; CHECK: {
+; CHECK: [[R3:v[0-9]+]].w = vadd([[R1]].w,[[R2]].w)
+; CHECK: }
+; CHECK: {
+; CHECK: [[R4:v[0-9]+]] = valign([[_:v[0-9]+]],[[R3]],{{.+}})
+; CHECK: }
+; CHECK: {
+; CHECK: [[R5:v[0-9]+]].w = vadd([[R3]].w,[[R4]].w)
+; CHECK: }
+; CHECK: {
+; CHECK: [[R6:v[0-9]+]] = valign([[_:v[0-9]+]],[[R5]],{{.+}})
+; CHECK: }
+; CHECK: {
+; CHECK: [[R7:v[0-9]+]].w = vadd([[R5]].w,[[R6]].w)
+; CHECK: }
+; CHECK: {
+; CHECK: [[R8:v[0-9]+]] = valign([[_:v[0-9]+]],[[R7]],{{.+}})
+; CHECK: }
+; CHECK: {
+; CHECK: [[R9:v[0-9]+]].w = vadd([[R7]].w,[[R8]].w)
+; CHECK: }
+; CHECK: {
+; CHECK: r0 = vextract([[R9]],{{.+}})
+; CHECK: }
+entry:
+ %r = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %vec)
+ ret i32 %r
+}
+
+define i32 @add_v16i32(<16 x i32> %vec) #0 {
+; CHECK-LABEL: add_v16i32:
+; CHECK: {
+; CHECK: [[R0:v[0-9]+]] = valign([[_:v[0-9]+]],v0,{{.+}})
+; CHECK: }
+; CHECK: {
+; CHECK: [[R1:v[0-9]+]].w = vadd(v0.w,[[R0]].w)
+; CHECK: }
+; CHECK: {
+; CHECK: [[R2:v[0-9]+]] = valign([[_:v[0-9]+]],[[R1]],{{.+}})
+; CHECK: }
+; CHECK: {
+; CHECK: [[R3:v[0-9]+]].w = vadd([[R1]].w,[[R2]].w)
+; CHECK: }
+; CHECK: {
+; CHECK: [[R4:v[0-9]+]] = valign([[_:v[0-9]+]],[[R3]],{{.+}})
+; CHECK: }
+; CHECK: {
+; CHECK: [[R5:v[0-9]+]].w = vadd([[R3]].w,[[R4]].w)
+; CHECK: }
+; CHECK: {
+; CHECK: [[R6:v[0-9]+]] = valign([[_:v[0-9]+]],[[R5]],{{.+}})
+; CHECK: }
+; CHECK: {
+; CHECK: [[R7:v[0-9]+]].w = vadd([[R5]].w,[[R6]].w)
+; CHECK: }
+; CHECK: {
+; CHECK: r0 = vextract([[R7]],{{.+}})
+; CHECK: }
+entry:
+ %r = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %vec)
+ ret i32 %r
+}
+
+define i32 @add_v8i32(<8 x i32> %vec) #0 {
+; CHECK-LABEL: add_v8i32:
+; CHECK: {
+; CHECK: r[[RS1:[0-9]+:[0-9]+]] = vaddw(r1:0,r5:4)
+; CHECK: r[[R6:[0-9]+:[0-9]+]] = memd(r29+#0)
+; CHECK: }
+; CHECK: {
+; CHECK: r[[RS2:[0-9]+:[0-9]+]] = vaddw(r3:2,r[[R6]])
+; CHECK: }
+; CHECK: {
+; CHECK: r[[RS3:[0-9]+:[0-9]+]] = vaddw(r[[RS1]],r[[RS2]])
+; CHECK: }
+; CHECK: {
+;; TODO: combine and double register add can be optimized to single register add.
+; CHECK: r[[RS4:[0-9]+:[0-9]+]] = combine(#0,r{{[0-9]+}})
+; CHECK: }
+; CHECK: {
+; CHECK: r1:0 = vaddw(r[[RS3]],r[[RS4]])
+entry:
+ %r = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %vec)
+ ret i32 %r
+}
+
+define i32 @add_v64i32(<64 x i32> %vec) #0 {
+; CHECK-LABEL: add_v64i32:
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+entry:
+ %r = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %vec)
+ ret i32 %r
+}
+
+;; Non-pow2 vectors are scalarized.
+
+define i32 @add_v12i32(<12 x i32> %vec) #0 {
+; CHECK-LABEL: add_v12i32:
+; CHECK: [[RS0:r[0-9]+]] = add(r0,r1)
+; CHECK: [[RS1:r[0-9]+]] += add([[RS0]],r{{[0-9]+}})
+; CHECK: [[RS2:r[0-9]+]] += add([[RS1]],r{{[0-9]+}})
+; CHECK: [[RS3:r[0-9]+]] += add([[RS2]],r{{[0-9]+}})
+; CHECK: [[RS4:r[0-9]+]] += add([[RS3]],r{{[0-9]+}})
+; CHECK: [[RS5:r[0-9]+]] += add([[RS4]],r{{[0-9]+}})
+entry:
+ %r = call i32 @llvm.vector.reduce.add.v12i32(<12 x i32> %vec)
+ ret i32 %r
+}
+
+define i32 @add_v3i32(<3 x i32> %vec) #0 {
+; CHECK-LABEL: add_v3i32:
+; CHECK: r{{[0-9]+}} += add(r{{[0-9]+}},r{{[0-9]+}})
+entry:
+ %r = call i32 @llvm.vector.reduce.add.v3i32(<3 x i32> %vec)
+ ret i32 %r
+}
+
+attributes #0 = { nounwind readnone "target-cpu"="hexagonv68" "target-features"="+hvx,+hvx-length128b" }
>From 62ccade3629778fa5d55577e3dda54787a66aebf Mon Sep 17 00:00:00 2001
From: Alexey Karyakin <akaryaki at qti.qualcomm.com>
Date: Mon, 29 Dec 2025 09:15:25 -0800
Subject: [PATCH 5/6] [Hexagon] NFC: Move vecreduce_add expansion to backend
This is a preliminary step before introducing vector combining for
full reductions in the Hexagon backend. Tell the generic expansion pass
that vecreduce_add will be handled by the backend and implement it.
Change-Id: Ic9ddaa9d6804fbe5d1d3230c69cf79ad8cac8db1
---
.../Target/Hexagon/HexagonISelLowering.cpp | 81 ++++++++++++++++++-
llvm/lib/Target/Hexagon/HexagonISelLowering.h | 4 +-
.../Hexagon/HexagonTargetTransformInfo.cpp | 8 ++
.../Hexagon/HexagonTargetTransformInfo.h | 2 +-
4 files changed, 90 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 6f1e52998996d..7591b59f6ef66 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -1506,6 +1506,8 @@ HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
MaxStoresPerMemset = 8;
MaxStoresPerMemsetOptSize = 4;
+ setTargetDAGCombine(ISD::VECREDUCE_ADD);
+
//
// Set up register classes.
//
@@ -3408,7 +3410,12 @@ HexagonTargetLowering::PerformDAGCombine(SDNode *N,
unsigned Opc = Op.getOpcode();
// Combining transformations applicable for arbitrary vector sizes.
- if (!DCI.isBeforeLegalizeOps()) {
+ if (DCI.isBeforeLegalizeOps()) {
+ switch (Opc) {
+ case ISD::VECREDUCE_ADD:
+ return expandVecReduceAdd(N, DCI.DAG);
+ }
+ } else {
switch (Opc) {
case ISD::VSELECT: {
// (vselect (xor x, ptrue), v0, v1) -> (vselect x, v1, v0)
@@ -3753,6 +3760,78 @@ EVT HexagonTargetLowering::getOptimalMemOpType(
return MVT::Other;
}
+// The helpers below are versions of llvm::getShuffleReduction and
+// llvm::getOrderedReduction, adapted to use during DAG passes and simplified as
+// follows:
+// - ICmp and FCmp are not handled;
+// - in every step in getShuffleReduction, the input is split into halves (not
+// pairwise).
+
+static SDValue getOrderedReduction(SDValue Vec, unsigned Op,
+ SelectionDAG &DAG) {
+ assert(Op != Instruction::ICmp && Op != Instruction::FCmp);
+
+ EVT VT = Vec.getValueType();
+ EVT EltT = VT.getVectorElementType();
+ unsigned VF = VT.getVectorNumElements();
+ assert(VF > 0 &&
+ "Reduction emission only supported for non-zero length vectors!");
+
+ SDLoc DL(Vec);
+ SDValue Result = DAG.getExtractVectorElt(DL, EltT, Vec, 0);
+ for (unsigned ExtractIdx = 1; ExtractIdx < VF; ++ExtractIdx) {
+ SDValue Ext = DAG.getExtractVectorElt(DL, EltT, Vec, ExtractIdx);
+ Result = DAG.getNode(Op, DL, EltT, {Result, Ext});
+ }
+
+ return Result;
+}
+
+static SDValue getShuffleReduction(SDValue Vec, unsigned Op,
+ SelectionDAG &DAG) {
+ assert(Op != Instruction::ICmp && Op != Instruction::FCmp);
+
+ EVT VT = Vec.getValueType();
+ unsigned VF = VT.getVectorNumElements();
+ if (VF == 0)
+ llvm_unreachable("Vector must be non-zero length");
+ // VF is a power of 2 so we can emit the reduction using log2(VF) shuffles
+ // and vector ops, reducing the set of values being computed by half each
+ // round.
+ assert(isPowerOf2_32(VF) &&
+ "Reduction emission only supported for pow2 vectors!");
+
+ SDLoc DL(Vec);
+ // TODO: Is it correct to create double-vector shuffle and fill 3/4 of it with
+ // undefs?
+ SmallVector<int, 32> ShuffleMask(VF);
+ for (unsigned i = VF; i > 1; i >>= 1) {
+ // Move the upper half of the vector to the lower half.
+ for (unsigned j = 0; j != i / 2; ++j)
+ ShuffleMask[j] = i / 2 + j;
+ // Fill the rest of the mask with undef.
+ std::fill(&ShuffleMask[i / 2], ShuffleMask.end(), -1);
+
+ SDValue Shuf =
+ DAG.getVectorShuffle(VT, DL, Vec, DAG.getUNDEF(VT), ShuffleMask);
+
+ Vec = DAG.getNode(Op, DL, VT, {Vec, Shuf});
+ }
+ // The result is in the first element of the vector.
+ return DAG.getExtractVectorElt(DL, VT.getVectorElementType(), Vec, 0);
+}
+
+SDValue HexagonTargetLowering::expandVecReduceAdd(SDNode *N,
+ SelectionDAG &DAG) const {
+ // Since we disabled automatic reduction expansion, generate log2 ladder code
+ // if the vector is of a power-of-two length.
+ SDValue Input = N->getOperand(0);
+ if (isPowerOf2_32(Input.getValueType().getVectorNumElements()))
+ return getShuffleReduction(Input, ISD::ADD, DAG);
+ // Otherwise, reduction will be scalarized.
+ return getOrderedReduction(Input, ISD::ADD, DAG);
+}
+
bool HexagonTargetLowering::allowsMemoryAccess(
LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const {
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.h b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
index f882fe03d465a..031d5f5cc34c5 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.h
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
@@ -517,9 +517,7 @@ class HexagonTargetLowering : public TargetLowering {
SDValue combineTruncateBeforeLegal(SDValue Op, DAGCombinerInfo &DCI) const;
SDValue combineConcatVectorsBeforeLegal(SDValue Op, DAGCombinerInfo & DCI)
const;
- SDValue combineVectorShuffleBeforeLegal(SDValue Op, DAGCombinerInfo & DCI)
- const;
-
+ SDValue expandVecReduceAdd(SDNode *N, SelectionDAG &DAG) const;
SDValue PerformHvxDAGCombine(SDNode * N, DAGCombinerInfo & DCI) const;
};
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 25ede7d262544..c2e9b3527e1ee 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -327,6 +327,14 @@ InstructionCost HexagonTTIImpl::getVectorInstrCost(
return 1;
}
+bool HexagonTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
+ switch (II->getIntrinsicID()) {
+ case Intrinsic::vector_reduce_add:
+ return false;
+ }
+ return true;
+}
+
bool HexagonTTIImpl::isLegalMaskedStore(Type *DataType, Align /*Alignment*/,
unsigned /*AddressSpace*/,
TTI::MaskKind /*MaskKind*/) const {
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index 0bd07a97ff3d5..25e1b5b5f6455 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -156,7 +156,7 @@ class HexagonTTIImpl final : public BasicTTIImplBase<HexagonTTIImpl> {
const Instruction *I = nullptr) const override {
return 1;
}
-
+ bool shouldExpandReduction(const IntrinsicInst *II) const override;
bool isLegalMaskedStore(Type *DataType, Align Alignment,
unsigned AddressSpace,
TTI::MaskKind MaskKind) const override;
>From a84e865518038afb85625a6f4643f42774b7326d Mon Sep 17 00:00:00 2001
From: Alexey Karyakin <akaryaki at qti.qualcomm.com>
Date: Tue, 30 Dec 2025 08:55:38 -0800
Subject: [PATCH 6/6] [Hexagon] Separate partial reduction from full and MLA
Full (vecreduce_add) and partial reductions (partial_reduce_U/S/SUMLA)
over multiplies which operands are extended (from i8 to i32 currently)
can be optimized by performing a partial reduction on original operands
using a vrmpy instruction.
If the reduction is suitable, separate a partial reduction into a new
operation, which will be lowered to vrmpy.
Change-Id: Id1de9f1f028d7160be1788771f2e52e8c4008295
---
.../Target/Hexagon/HexagonISelLowering.cpp | 12 +-
llvm/lib/Target/Hexagon/HexagonISelLowering.h | 9 +-
.../Target/Hexagon/HexagonISelLoweringHVX.cpp | 261 +++++++++++++++++-
llvm/test/CodeGen/Hexagon/hvx-full-reduce.ll | 145 ++++++++++
.../CodeGen/Hexagon/hvx-partial-reduce.ll | 54 ++++
5 files changed, 472 insertions(+), 9 deletions(-)
create mode 100644 llvm/test/CodeGen/Hexagon/hvx-full-reduce.ll
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 7591b59f6ef66..68c50eaf52a1d 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -3413,7 +3413,17 @@ HexagonTargetLowering::PerformDAGCombine(SDNode *N,
if (DCI.isBeforeLegalizeOps()) {
switch (Opc) {
case ISD::VECREDUCE_ADD:
- return expandVecReduceAdd(N, DCI.DAG);
+ if (SDValue V = splitVecReduceAdd(N, DCI.DAG))
+ return V;
+ if (SDValue V = expandVecReduceAdd(N, DCI.DAG))
+ return V;
+ return SDValue();
+ case ISD::PARTIAL_REDUCE_SMLA:
+ case ISD::PARTIAL_REDUCE_UMLA:
+ case ISD::PARTIAL_REDUCE_SUMLA:
+ if (SDValue V = splitExtendingPartialReduceMLA(N, DCI.DAG))
+ return V;
+ return SDValue();
}
} else {
switch (Opc) {
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.h b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
index 031d5f5cc34c5..9fb2bfb024224 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.h
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
@@ -485,6 +485,7 @@ class HexagonTargetLowering : public TargetLowering {
SDValue LowerHvxIntToFp(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerHvxPred32ToFp(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerHvxPred64ToFp(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerHvxPartialReduceMLA(SDValue Op, SelectionDAG &DAG) const;
SDValue ExpandHvxFpToInt(SDValue Op, SelectionDAG &DAG) const;
SDValue ExpandHvxIntToFp(SDValue Op, SelectionDAG &DAG) const;
@@ -518,7 +519,13 @@ class HexagonTargetLowering : public TargetLowering {
SDValue combineConcatVectorsBeforeLegal(SDValue Op, DAGCombinerInfo & DCI)
const;
SDValue expandVecReduceAdd(SDNode *N, SelectionDAG &DAG) const;
- SDValue PerformHvxDAGCombine(SDNode * N, DAGCombinerInfo & DCI) const;
+ SDValue createExtendingPartialReduceMLA(
+ unsigned Opcode, EVT AccEltType, unsigned AccNumElements, EVT InputType,
+ const SDValue &A, const SDValue &B, unsigned &RemainingReductionRatio,
+ const SDLoc &DL, SelectionDAG &DAG) const;
+ SDValue splitVecReduceAdd(SDNode *N, SelectionDAG &DAG) const;
+ SDValue splitExtendingPartialReduceMLA(SDNode *N, SelectionDAG &DAG) const;
+ SDValue PerformHvxDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const;
};
} // end namespace llvm
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
index bebff0ca4e7bd..ad499a3cc50d7 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
@@ -40,6 +40,8 @@ static const MVT LegalW64[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
static const MVT LegalV128[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
static const MVT LegalW128[] = { MVT::v256i8, MVT::v128i16, MVT::v64i32 };
+static const unsigned MaxExpandMLA = 8;
+
static std::tuple<unsigned, unsigned, unsigned> getIEEEProperties(MVT Ty) {
// For a float scalar type, return (exp-bits, exp-bias, fraction-bits)
MVT ElemTy = Ty.getScalarType();
@@ -499,19 +501,67 @@ HexagonTargetLowering::initializeHVXLowering() {
setTargetDAGCombine({ISD::CONCAT_VECTORS, ISD::TRUNCATE, ISD::VSELECT});
- // Partial reductions.
+ setTargetDAGCombine({ISD::PARTIAL_REDUCE_SMLA, ISD::PARTIAL_REDUCE_UMLA,
+ ISD::PARTIAL_REDUCE_SUMLA});
+
+ // Partial MLA reductions.
{
static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA,
ISD::PARTIAL_REDUCE_UMLA,
ISD::PARTIAL_REDUCE_SUMLA};
- auto HvxType = [=](MVT ScalarT) {
- return MVT::getVectorVT(ScalarT, Subtarget.getVectorLength() * 8 /
- ScalarT.getSizeInBits());
+ auto HvxType = [=](MVT ScalarT, unsigned Factor = 1) {
+ return MVT::getVectorVT(ScalarT, Subtarget.getVectorLength() * Factor *
+ 8 / ScalarT.getSizeInBits());
+ };
+
+ // Tuple of (Acc element type, input element type, vector pair).
+ // The assumption is both the input and reduction result are of the same
+ // size so the reduction ratio is the same as the ratio of element type
+ // sizes. This may not hold for all available instructions.
+ typedef std::tuple<MVT, MVT, bool> ReductionSignature;
+
+ static const std::vector<ReductionSignature> NativeReductions = {
+ {MVT::i32, MVT::i8, false},
};
- setPartialReduceMLAAction(MLAOps, HvxType(MVT::i32), HvxType(MVT::i8),
- Legal);
+ for (const auto &R : NativeReductions) {
+
+ MVT AccType = std::get<0>(R);
+ MVT InputType = std::get<1>(R);
+ unsigned Factor = std::get<2>(R) ? 2 : 1;
+
+ // The native size is legal.
+ setPartialReduceMLAAction(MLAOps, HvxType(AccType), HvxType(InputType),
+ Legal);
+
+ // Allow custom partial MLA reductions on larger vectors than legally
+ // supported. These reduction must be declared as Custom (or Legal)
+ // for foldPartialReduceMLAMulOp() to fold the multiply by one pattern
+ // inserted when the partial reduction intrinsic is converted to
+ // PARTIAL_REDUCE_U/S/SUMLA. Otherwise, the Split action will apply
+ // on the original pattern, including the extensions and multiplies,
+ // which will make it impossible to match.
+ // There are two independent ways to extend the
+ // input size: 1. to concatenate the result - output vector is
+ // proportionally extended, 2) to reduce the result - the output vector
+ // size stays the same. We limit allowed combinations so that the total
+ // number of generated reduction instructions is limited by a constant
+ // number. This limit is arbitrary and can be revised. On one hand, it is
+ // convenient to have more choices; on the other hand, there is a
+ // diminishing benefit of very long sequences, which should probably be
+ // written as loops instead.
+ for (unsigned ConcatFactor = 1; ConcatFactor <= MaxExpandMLA;
+ ConcatFactor <<= 1)
+ for (unsigned ReductionFactor = 1; ReductionFactor <= MaxExpandMLA;
+ ReductionFactor <<= 1)
+ if (ConcatFactor * ReductionFactor != 1 &&
+ ConcatFactor * ReductionFactor <= MaxExpandMLA)
+ setPartialReduceMLAAction(
+ MLAOps, HvxType(AccType, Factor * ConcatFactor),
+ HvxType(InputType, Factor * ConcatFactor * ReductionFactor),
+ Custom);
+ }
}
}
@@ -3564,7 +3614,12 @@ HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const {
case HexagonISD::SMUL_LOHI:
case HexagonISD::UMUL_LOHI:
case HexagonISD::USMUL_LOHI: return LowerHvxMulLoHi(Op, DAG);
- }
+
+ case ISD::PARTIAL_REDUCE_SMLA:
+ case ISD::PARTIAL_REDUCE_UMLA:
+ case ISD::PARTIAL_REDUCE_SUMLA:
+ return LowerHvxPartialReduceMLA(Op, DAG);
+ }
#ifndef NDEBUG
Op.dumpr(&DAG);
#endif
@@ -3905,6 +3960,198 @@ HexagonTargetLowering::combineConcatVectorsBeforeLegal(
return DAG.getVectorShuffle(LongTy, dl, Cat, DAG.getUNDEF(LongTy), LongMask);
}
+// Create the inner partial reduction MLA that can be efficiently lowered. This
+// function is used by partial and full reductions.
+SDValue HexagonTargetLowering::createExtendingPartialReduceMLA(
+ unsigned Opcode, EVT AccEltType, unsigned AccNumElements, EVT InputType,
+ const SDValue &A, const SDValue &B, unsigned &RemainingReductionRatio,
+ const SDLoc &DL, SelectionDAG &DAG) const {
+ const auto &Subtarget = DAG.getSubtarget<HexagonSubtarget>();
+ if (!Subtarget.useHVXOps())
+ return SDValue();
+
+ EVT InputEltType = InputType.getVectorElementType();
+
+ // Find if an optimized instruction for the sub-reduction is available.
+ unsigned NativeRatio;
+ if (AccEltType == MVT::i32 && InputEltType == MVT::i8)
+ NativeRatio = 4;
+ else
+ return SDValue();
+
+ // We only handle the case when additional reduction will be needed, i.e.
+ // input is longer by a larger factor than the result.
+ ElementCount InputEC = InputType.getVectorElementCount();
+ if (!InputEC.isKnownMultipleOf(AccNumElements * NativeRatio))
+ return SDValue();
+
+ unsigned InputNumElements = InputEC.getFixedValue();
+ RemainingReductionRatio = InputNumElements / (AccNumElements * NativeRatio);
+ if (RemainingReductionRatio == 1)
+ return SDValue();
+
+ // Create a reduction by the natively supported factor.
+ EVT IntermediateType = EVT::getVectorVT(*DAG.getContext(), AccEltType,
+ InputNumElements / NativeRatio);
+
+ SDValue Zero = DAG.getConstant(0, DL, IntermediateType);
+ return DAG.getNode(Opcode, DL, IntermediateType, Zero, A, B);
+}
+
+static bool DetectExtendingMultiply(const SDValue &N, EVT ScalarType,
+ unsigned &Opcode, SDValue &A, SDValue &B) {
+ SDValue Mul = N;
+ EVT AccType = Mul.getValueType(); // Vector input type after extension.
+ if (ScalarType != AccType.getVectorElementType())
+ return false;
+ bool swap = false;
+ if (Mul->getOpcode() != ISD::MUL)
+ return false;
+ A = Mul->getOperand(0);
+ B = Mul->getOperand(1);
+ if (A.getOpcode() == ISD::ZERO_EXTEND) {
+ if (B.getOpcode() == ISD::ZERO_EXTEND)
+ Opcode = ISD::PARTIAL_REDUCE_UMLA;
+ else if (B.getOpcode() == ISD::SIGN_EXTEND) {
+ swap = true;
+ Opcode = ISD::PARTIAL_REDUCE_SUMLA;
+ } else
+ return false;
+ } else if (A.getOpcode() == ISD::SIGN_EXTEND) {
+ if (B.getOpcode() == ISD::ZERO_EXTEND)
+ Opcode = ISD::PARTIAL_REDUCE_SUMLA;
+ else if (B.getOpcode() == ISD::SIGN_EXTEND)
+ Opcode = ISD::PARTIAL_REDUCE_SMLA;
+ else
+ return false;
+ } else
+ return false;
+
+ // Get multiplication arguments before extension.
+ A = A->getOperand(0);
+ B = B->getOperand(0);
+ if (A.getValueType() != B.getValueType())
+ return false;
+
+ if (swap)
+ std::swap(A, B);
+
+ return true;
+}
+
+SDValue HexagonTargetLowering::splitVecReduceAdd(SDNode *N,
+ SelectionDAG &DAG) const {
+ if (!Subtarget.useHVXOps())
+ return SDValue();
+
+ EVT ScalarType = N->getValueType(0);
+ unsigned Opcode;
+ SDValue A, B;
+ if (!DetectExtendingMultiply(N->getOperand(0), ScalarType, Opcode, A, B))
+ return SDValue();
+
+ SDLoc DL(N);
+ unsigned RemainingReductionRatio;
+ SDValue Partial =
+ createExtendingPartialReduceMLA(Opcode, ScalarType, 1, A.getValueType(),
+ A, B, RemainingReductionRatio, DL, DAG);
+ if (!Partial)
+ return SDValue();
+
+ // We could have inserted a trivial MLA and rely on the folding action,
+ // similar to how vector_partial_reduce_add is lowered to an MLA in
+ // SelectionDAGBuilder. However, we just replace the final result since we
+ // have analyzed the input completely.
+ return DAG.getNode(ISD::VECREDUCE_ADD, DL, ScalarType, Partial);
+}
+
+// When possible, separate an MLA reduction with extended operands but
+// unsupported reduction factor into an extending partial reduction that
+// can be efficiently lowered, and a follow-up partial reduction.
+// partial_reduce_mla(a, x, y) ->
+// partial_reduce_mla(a, partial_reduce_mla(0, x, y), 1)
+SDValue
+HexagonTargetLowering::splitExtendingPartialReduceMLA(SDNode *N,
+ SelectionDAG &DAG) const {
+ if (!Subtarget.useHVXOps())
+ return SDValue();
+
+ SDValue Acc = N->getOperand(0);
+ SDValue A = N->getOperand(1);
+ SDValue B = N->getOperand(2);
+ if (A.getValueType() != B.getValueType())
+ return SDValue();
+
+ // The types should be declared as custom, but do not split already legal
+ // operation.
+ EVT AccType = Acc.getValueType();
+ EVT InputType = A.getValueType();
+ if (getPartialReduceMLAAction(N->getOpcode(), AccType, InputType) != Custom)
+ return SDValue();
+
+ SDLoc DL(N);
+ unsigned RemainingReductionRatio;
+ SDValue Partial = createExtendingPartialReduceMLA(
+ N->getOpcode(), AccType.getVectorElementType(),
+ AccType.getVectorNumElements(), InputType, A, B, RemainingReductionRatio,
+ DL, DAG);
+ if (!Partial)
+ return SDValue();
+ assert(RemainingReductionRatio <= MaxExpandMLA);
+
+ // Create the reduction for the remaining ratio.
+ EVT IntermediateType = Partial->getOperand(0).getValueType();
+ SDValue One = DAG.getConstant(1, DL, IntermediateType);
+ return DAG.getNode(N->getOpcode() == ISD::PARTIAL_REDUCE_UMLA
+ ? ISD::PARTIAL_REDUCE_UMLA
+ : ISD::PARTIAL_REDUCE_SUMLA,
+ DL, AccType, Acc, Partial, One);
+}
+
+SDValue
+HexagonTargetLowering::LowerHvxPartialReduceMLA(SDValue Op,
+ SelectionDAG &DAG) const {
+ const SDLoc &DL(Op);
+ SDValue Acc = Op.getOperand(0);
+ SDValue A = Op.getOperand(1);
+ SDValue B = Op.getOperand(2);
+
+ // Split the input vectors into units of one HVX vector length.
+ unsigned HwVectorSizeInBits = Subtarget.getVectorLength() * 8;
+
+ EVT AccType = Acc.getValueType();
+ EVT AccEltType = AccType.getVectorElementType();
+ unsigned AccSubvectorNumElements =
+ HwVectorSizeInBits / AccEltType.getSizeInBits();
+ EVT AccSubvectorType =
+ EVT::getVectorVT(*DAG.getContext(), AccEltType, AccSubvectorNumElements);
+
+ EVT InputType = A.getValueType();
+ assert(InputType.getSizeInBits() % HwVectorSizeInBits == 0);
+ EVT InputEltType = InputType.getVectorElementType();
+ unsigned InputSubvectorNumElements =
+ HwVectorSizeInBits / InputEltType.getSizeInBits();
+ EVT InputSubvectorType = EVT::getVectorVT(*DAG.getContext(), InputEltType,
+ InputSubvectorNumElements);
+
+ unsigned SubvectorNum = InputType.getFixedSizeInBits() / HwVectorSizeInBits;
+ SmallVector<SDValue, MaxExpandMLA> Subvectors;
+
+ for (unsigned I = 0; I != SubvectorNum; ++I) {
+ SDValue SubvectorAcc = DAG.getExtractSubvector(DL, AccSubvectorType, Acc,
+ I * AccSubvectorNumElements);
+ SDValue SubvectorA = DAG.getExtractSubvector(DL, InputSubvectorType, A,
+ I * InputSubvectorNumElements);
+ SDValue SubvectorB = DAG.getExtractSubvector(DL, InputSubvectorType, B,
+ I * InputSubvectorNumElements);
+ SDValue SubvectorMLA = DAG.getNode(Op.getOpcode(), DL, AccSubvectorType,
+ SubvectorAcc, SubvectorA, SubvectorB);
+ Subvectors.push_back(SubvectorMLA);
+ }
+
+ return DAG.getNode(ISD::CONCAT_VECTORS, DL, AccType, Subvectors);
+}
+
SDValue
HexagonTargetLowering::PerformHvxDAGCombine(SDNode *N, DAGCombinerInfo &DCI)
const {
diff --git a/llvm/test/CodeGen/Hexagon/hvx-full-reduce.ll b/llvm/test/CodeGen/Hexagon/hvx-full-reduce.ll
new file mode 100644
index 0000000000000..c49b7305725b9
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/hvx-full-reduce.ll
@@ -0,0 +1,145 @@
+; RUN: llc -mtriple=hexagon < %s | FileCheck %s
+
+define i32 @full_reduce_i32_128i8_uu(<128 x i8> %x, <128 x i8> %y) #0 {
+; CHECK-LABEL: full_reduce_i32_128i8_uu:
+; CHECK: [[A:v[0-9]+]] = vxor([[Z:v[0-9]+]],[[Z]])
+; CHECK: [[A]].uw += vrmpy(v0.ub,v1.ub)
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: vextract
+ %x.wide = zext <128 x i8> %x to <128 x i32>
+ %y.wide = zext <128 x i8> %y to <128 x i32>
+ %m = mul nuw nsw <128 x i32> %x.wide, %y.wide
+ %reduce = tail call i32 @llvm.vector.reduce.add.v128i32(<128 x i32> %m)
+ ret i32 %reduce
+}
+
+define i32 @full_reduce_i32_128i8_su(<128 x i8> %x, <128 x i8> %y) #0 {
+; CHECK-LABEL: full_reduce_i32_128i8_su:
+; CHECK: [[A:v[0-9]+]] = vxor([[Z:v[0-9]+]],[[Z]])
+; CHECK: [[A]].w += vrmpy(v1.ub,v0.b)
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: vextract
+ %x.wide = sext <128 x i8> %x to <128 x i32>
+ %y.wide = zext <128 x i8> %y to <128 x i32>
+ %m = mul nuw nsw <128 x i32> %x.wide, %y.wide
+ %reduce = tail call i32 @llvm.vector.reduce.add.v128i32(<128 x i32> %m)
+ ret i32 %reduce
+}
+
+define i32 @full_reduce_i32_128i8_us(<128 x i8> %x, <128 x i8> %y) #0 {
+; CHECK-LABEL: full_reduce_i32_128i8_us:
+; CHECK: [[A:v[0-9]+]] = vxor([[Z:v[0-9]+]],[[Z]])
+; CHECK: [[A]].w += vrmpy(v0.ub,v1.b)
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: vextract
+ %x.wide = zext <128 x i8> %x to <128 x i32>
+ %y.wide = sext <128 x i8> %y to <128 x i32>
+ %m = mul nuw nsw <128 x i32> %x.wide, %y.wide
+ %reduce = tail call i32 @llvm.vector.reduce.add.v128i32(<128 x i32> %m)
+ ret i32 %reduce
+}
+
+define i32 @full_reduce_i32_128i8_ss(<128 x i8> %x, <128 x i8> %y) #0 {
+; CHECK-LABEL: full_reduce_i32_128i8_ss:
+; CHECK: [[A:v[0-9]+]] = vxor([[Z:v[0-9]+]],[[Z]])
+; CHECK: [[A]].w += vrmpy(v0.b,v1.b)
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: vextract
+ %x.wide = sext <128 x i8> %x to <128 x i32>
+ %y.wide = sext <128 x i8> %y to <128 x i32>
+ %m = mul nuw nsw <128 x i32> %x.wide, %y.wide
+ %reduce = tail call i32 @llvm.vector.reduce.add.v128i32(<128 x i32> %m)
+ ret i32 %reduce
+}
+
+;; Double-vector input.
+
+define i32 @full_reduce_i32_256i8(<256 x i8> %x, <256 x i8> %y) #0 {
+; CHECK-LABEL: full_reduce_i32_256i8:
+; CHECK: vrmpy
+; CHECK: vrmpy
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+ %x.wide = zext <256 x i8> %x to <256 x i32>
+ %y.wide = zext <256 x i8> %y to <256 x i32>
+ %m = mul nuw nsw <256 x i32> %x.wide, %y.wide
+ %reduce = tail call i32 @llvm.vector.reduce.add.v256i32(<256 x i32> %m)
+ ret i32 %reduce
+}
+
+;; Maximum handled vector size.
+
+define i32 @full_reduce_i32_1024i8(<1024 x i8> %x, <1024 x i8> %y) #0 {
+; CHECK-LABEL: full_reduce_i32_1024i8:
+; CHECK: vrmpy
+; CHECK: vrmpy
+; CHECK: vrmpy
+; CHECK: vrmpy
+; CHECK: vrmpy
+; CHECK: vrmpy
+; CHECK: vrmpy
+; CHECK: vrmpy
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+; CHECK: valign
+; CHECK: vadd
+ %x.wide = zext <1024 x i8> %x to <1024 x i32>
+ %y.wide = zext <1024 x i8> %y to <1024 x i32>
+ %m = mul nuw nsw <1024 x i32> %x.wide, %y.wide
+ %reduce = tail call i32 @llvm.vector.reduce.add.v1024i32(<1024 x i32> %m)
+ ret i32 %reduce
+}
+
+attributes #0 = { nounwind readnone "target-cpu"="hexagonv68" "target-features"="+hvx,+hvx-length128b" }
diff --git a/llvm/test/CodeGen/Hexagon/hvx-partial-reduce.ll b/llvm/test/CodeGen/Hexagon/hvx-partial-reduce.ll
index 0165d5933a7d1..b2c2c1893ea44 100644
--- a/llvm/test/CodeGen/Hexagon/hvx-partial-reduce.ll
+++ b/llvm/test/CodeGen/Hexagon/hvx-partial-reduce.ll
@@ -92,6 +92,60 @@ define <32 x i32> @partial_reduce_ss_128(<32 x i32> %acc, <128 x i8> %x, <128 x
ret <32 x i32> %partial.reduce
}
+;; Multiple-size inputs, same output size.
+define <32 x i32> @partial_reduce_uu_32xi32_256xi8(<32 x i32> %acc, <256 x i8> %x, <256 x i8> %y) #1 {
+; CHECK-LABEL: partial_reduce_uu_32xi32_256xi8:
+; CHECK-HVX: [[R1:v[0-9]+]].uw += vrmpy({{v[0-9]+}}.ub,{{v[0-9]+}}.ub)
+; CHECK-HVX: [[R2:v[0-9]+]].uw += vrmpy({{v[0-9]+}}.ub,{{v[0-9]+}}.ub)
+; CHECK-HVX: [[R3:v[0-9]+]].w = vadd(v0.w,[[R1]].w)
+; CHECK-HVX: v0.w = vadd([[R2]].w,[[R3]].w)
+; CHECK-NO-HVX: {{r[0-9]+}} += mpyi
+ %x.ext = zext <256 x i8> %x to <256 x i32>
+ %y.ext = zext <256 x i8> %y to <256 x i32>
+ %m = mul nuw nsw <256 x i32> %x.ext, %y.ext
+ %partial.reduce = tail call <32 x i32> @llvm.vector.partial.reduce.add.v32i32.v256i32(<32 x i32> %acc, <256 x i32> %m)
+ ret <32 x i32> %partial.reduce
+}
+
+define <32 x i32> @partial_reduce_uu_32xi32_1024xi8(<32 x i32> %acc, <1024 x i8> %x, <1024 x i8> %y) #1 {
+; CHECK-LABEL: partial_reduce_uu_32xi32_1024xi8:
+; CHECK-HVX-DAG: vrmpy
+; CHECK-HVX-DAG: vadd
+; CHECK-HVX-DAG: vrmpy
+; CHECK-HVX-DAG: vadd
+; CHECK-HVX-DAG: vrmpy
+; CHECK-HVX-DAG: vadd
+; CHECK-HVX-DAG: vrmpy
+; CHECK-HVX-DAG: vadd
+; CHECK-HVX-DAG: vrmpy
+; CHECK-HVX-DAG: vadd
+; CHECK-HVX-DAG: vrmpy
+; CHECK-HVX-DAG: vadd
+; CHECK-HVX-DAG: vrmpy
+; CHECK-HVX-DAG: vadd
+; CHECK-HVX-DAG: vrmpy
+; CHECK-HVX-DAG: vadd
+; CHECK-NO-HVX: {{r[0-9]+}} += mpyi
+ %x.ext = zext <1024 x i8> %x to <1024 x i32>
+ %y.ext = zext <1024 x i8> %y to <1024 x i32>
+ %m = mul nuw nsw <1024 x i32> %x.ext, %y.ext
+ %partial.reduce = tail call <32 x i32> @llvm.vector.partial.reduce.add.v32i32.v1024i32(<32 x i32> %acc, <1024 x i32> %m)
+ ret <32 x i32> %partial.reduce
+}
+
+define <256 x i32> @partial_reduce_uu_64xi32_1024xi8(<256 x i32> %acc, <1024 x i8> %x, <1024 x i8> %y) #1 {
+; CHECK-LABEL: partial_reduce_uu_64xi32_1024xi8:
+; CHECK-HVX-COUNT-8: vrmpy
+; CHECK-HVX-NOT: vadd
+; CHECK-NO-HVX: {{r[0-9]+}} += mpyi
+; CHECK-HVX: dealloc_return
+ %x.ext = zext <1024 x i8> %x to <1024 x i32>
+ %y.ext = zext <1024 x i8> %y to <1024 x i32>
+ %m = mul nuw nsw <1024 x i32> %x.ext, %y.ext
+ %partial.reduce = tail call <256 x i32> @llvm.vector.partial.reduce.add.v32i32.v1024i32(<256 x i32> %acc, <1024 x i32> %m)
+ ret <256 x i32> %partial.reduce
+}
+
;; Check for vector size that do not match an available vrmpy (2x reduction).
define <64 x i32> @partial_reduce_unsupported(<64 x i32> %acc, <128 x i8> %x, <128 x i8> %y) #1 {
; CHECK-LABEL: partial_reduce_unsupported:
More information about the llvm-commits
mailing list