[llvm] [RISCV][P-ext] Add codegen for packed reduction sum (PR #206004)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 26 01:06:09 PDT 2026
https://github.com/sihuan created https://github.com/llvm/llvm-project/pull/206004
The predsum/predsumu instructions reduce a packed vector to a scalar:
each element is sign- or zero-extended to the accumulator width and
added to the scalar accumulator rs2.
This patch adds the llvm.riscv.predsum and predsumu intrinsics along
with isel patterns for target-legal results (predsum.bs/hs/ws, plus
the paired predsum.dbs/dhs for a GPRPair source on RV32).
The remaining combinations have a target-illegal result and are split
in ReplaceNodeResults:
* On RV64: An i32 accumulator reduces at i64 and is then truncated
(the 32-bit source is zero-extended first).
* On RV32: An i64 accumulator reduces to a 32-bit partial sum and is
widening-accumulated via wadda/waddau.
>From a7cb810b4118a1d57e430dbc603d3d46a7984fb6 Mon Sep 17 00:00:00 2001
From: SiHuaN <liyongtai at iscas.ac.cn>
Date: Fri, 26 Jun 2026 03:49:53 +0000
Subject: [PATCH] [RISCV][P-ext] Add codegen for packed reduction sum
The predsum/predsumu instructions reduce a packed vector to a scalar:
each element is sign- or zero-extended to the accumulator width and
added to the scalar accumulator rs2.
This patch adds the llvm.riscv.predsum and predsumu intrinsics along
with isel patterns for target-legal results (predsum.bs/hs/ws, plus
the paired predsum.dbs/dhs for a GPRPair source on RV32).
The remaining combinations have a target-illegal result and are split
in ReplaceNodeResults:
* On RV64: An i32 accumulator reduces at i64 and is then truncated
(the 32-bit source is zero-extended first).
* On RV32: An i64 accumulator reduces to a 32-bit partial sum and is
widening-accumulated via wadda/waddau.
---
llvm/include/llvm/IR/IntrinsicsRISCV.td | 8 +
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 51 +++++++
llvm/lib/Target/RISCV/RISCVInstrInfoP.td | 35 +++++
llvm/test/CodeGen/RISCV/rvp-simd-32.ll | 65 ++++++++
llvm/test/CodeGen/RISCV/rvp-simd-64.ll | 155 ++++++++++++++++++++
5 files changed, 314 insertions(+)
diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td
index 531f48c38622e..f201c0e38e94e 100644
--- a/llvm/include/llvm/IR/IntrinsicsRISCV.td
+++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td
@@ -2068,6 +2068,14 @@ class RVPBinaryIntrinsic
def int_riscv_pssa : RVPBinaryIntrinsic;
def int_riscv_paas : RVPBinaryIntrinsic;
def int_riscv_pasa : RVPBinaryIntrinsic;
+
+ // Packed Reduction Sum.
+ class RVPReductionIntrinsic
+ : DefaultAttrsIntrinsic<[llvm_anyint_ty],
+ [llvm_anyvector_ty, LLVMMatchType<0>],
+ [IntrNoMem, IntrSpeculatable]>;
+ def int_riscv_predsum : RVPReductionIntrinsic;
+ def int_riscv_predsumu : RVPReductionIntrinsic;
} // TargetPrefix = "riscv"
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index e5ce36afb7b2f..ff14ecbd4917c 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -637,6 +637,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
{ISD::UADDSAT, ISD::SADDSAT, ISD::USUBSAT, ISD::SSUBSAT}, P64VecVTs,
Legal);
setOperationAction(ISD::INTRINSIC_WO_CHAIN, P64VecVTs, Legal);
+ setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU}, P64VecVTs, Legal);
setOperationAction({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX},
P64VecVTs, Legal);
@@ -15814,6 +15815,56 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
DAG.getVectorIdxConstant(0, DL)));
return;
}
+ case Intrinsic::riscv_predsum:
+ case Intrinsic::riscv_predsumu: {
+ bool IsSigned = IntNo == Intrinsic::riscv_predsum;
+ SDValue Vec = N->getOperand(1);
+ MVT VecVT = Vec.getSimpleValueType();
+ auto Ext = [&](SDValue V) {
+ return DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL,
+ MVT::i64, V);
+ };
+ auto RedSum = [&](MVT VT, SDValue V, SDValue Acc) {
+ return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, N->getOperand(0), V,
+ Acc);
+ };
+
+ // RV32: i64 accumulator. Reduce to a 32-bit partial sum, then
+ // widening-accumulate into i64 via wadda/waddau (v2i32 uses wadda alone).
+ if (!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64) {
+ SDValue Acc = N->getOperand(2);
+ SDValue Res;
+ if (VecVT == MVT::v2i32) {
+ Res = DAG.getNode(ISD::ADD, DL, MVT::i64, Acc,
+ Ext(DAG.getExtractVectorElt(DL, MVT::i32, Vec, 0)));
+ Res = DAG.getNode(ISD::ADD, DL, MVT::i64, Res,
+ Ext(DAG.getExtractVectorElt(DL, MVT::i32, Vec, 1)));
+ } else {
+ // The paired predsum.dbs/dhs computes the 32-bit element sum.
+ SDValue Partial =
+ RedSum(MVT::i32, Vec, DAG.getConstant(0, DL, MVT::i32));
+ Res = DAG.getNode(ISD::ADD, DL, MVT::i64, Acc, Ext(Partial));
+ }
+ Results.push_back(Res);
+ return;
+ }
+
+ // RV64: i32 accumulator. Reduce at i64 (XLEN), then truncate.
+ if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
+ return;
+
+ // Zero the upper lanes (zext.w) so they don't contribute to the sum.
+ if (VecVT == MVT::v4i8 || VecVT == MVT::v2i16) {
+ MVT WideVT = VecVT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16;
+ SDValue Wide = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64,
+ DAG.getBitcast(MVT::i32, Vec));
+ Vec = DAG.getBitcast(WideVT, Wide);
+ }
+
+ SDValue Res = RedSum(MVT::i64, Vec, Ext(N->getOperand(2)));
+ Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
+ return;
+ }
case Intrinsic::riscv_orc_b:
case Intrinsic::riscv_brev8:
case Intrinsic::riscv_sha256sig0:
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
index 83095ae1985b4..6acdcb8379730 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
@@ -1754,6 +1754,17 @@ class PatGprPairGprPair<SDPatternOperator OpNode, RVInst Inst, ValueType vt>
: Pat<(vt (OpNode (vt GPRPair:$rs1), (vt GPRPair:$rs2))),
(Inst GPRPair:$rs1, GPRPair:$rs2)>;
+// Reduction sum. PatRedSum is the single-GPR source (predsum.bs/hs/ws), result
+// XLenVT; PatRedSumPair is the RV32 paired source (predsum.dbs/dhs), result i32.
+// Target-illegal results (i32 accumulator on RV64, i64 on RV32) don't reach
+// isel and are split in ReplaceNodeResults.
+class PatRedSum<SDPatternOperator OpNode, RVInst Inst, ValueType VecVT>
+ : Pat<(XLenVT (OpNode (VecVT GPR:$rs1), (XLenVT GPR:$rs2))),
+ (Inst GPR:$rs1, GPR:$rs2)>;
+class PatRedSumPair<SDPatternOperator OpNode, RVInst Inst, ValueType VecVT>
+ : Pat<(i32 (OpNode (VecVT GPRPair:$rs1), (i32 GPR:$rs2))),
+ (Inst GPRPair:$rs1, GPR:$rs2)>;
+
class PatGprPairImm<SDPatternOperator OpNode, RVInst Inst,
SDPatternOperator ImmType, ValueType vt>
: Pat<(vt (OpNode (vt GPRPair:$rs1), ImmType:$imm)),
@@ -2391,6 +2402,18 @@ let append Predicates = [IsRV32] in {
def : PatGprPairGprPair<setult, PMSLTU_DH, v4i16>;
def : PatGprPairGprPair<setult, PMSLTU_DW, v2i32>;
+ // 8-bit reduction sum patterns
+ def : PatRedSum<int_riscv_predsum, PREDSUM_BS, v4i8>;
+ def : PatRedSum<int_riscv_predsumu, PREDSUMU_BS, v4i8>;
+ def : PatRedSumPair<int_riscv_predsum, PREDSUM_DBS, v8i8>;
+ def : PatRedSumPair<int_riscv_predsumu, PREDSUMU_DBS, v8i8>;
+
+ // 16-bit reduction sum patterns
+ def : PatRedSum<int_riscv_predsum, PREDSUM_HS, v2i16>;
+ def : PatRedSum<int_riscv_predsumu, PREDSUMU_HS, v2i16>;
+ def : PatRedSumPair<int_riscv_predsum, PREDSUM_DHS, v4i16>;
+ def : PatRedSumPair<int_riscv_predsumu, PREDSUMU_DHS, v4i16>;
+
// [s|u]min/[s|u]max patterns
def : PatGprPairGprPair<smin, PMIN_DB, v8i8>;
def : PatGprPairGprPair<umin, PMINU_DB, v8i8>;
@@ -2600,6 +2623,18 @@ let append Predicates = [IsRV64] in {
def : PatGprGpr<setlt, PMSLT_W, v2i32>;
def : PatGprGpr<setult, PMSLTU_W, v2i32>;
+ // 8-bit reduction sum patterns
+ def : PatRedSum<int_riscv_predsum, PREDSUM_BS, v8i8>;
+ def : PatRedSum<int_riscv_predsumu, PREDSUMU_BS, v8i8>;
+
+ // 16-bit reduction sum patterns
+ def : PatRedSum<int_riscv_predsum, PREDSUM_HS, v4i16>;
+ def : PatRedSum<int_riscv_predsumu, PREDSUMU_HS, v4i16>;
+
+ // 32-bit reduction sum patterns
+ def : PatRedSum<int_riscv_predsum, PREDSUM_WS, v2i32>;
+ def : PatRedSum<int_riscv_predsumu, PREDSUMU_WS, v2i32>;
+
// 32-bit [s|u]min/[s|u]max patterns
def : PatGprGpr<smin, PMIN_W, v2i32>;
def : PatGprGpr<umin, PMINU_W, v2i32>;
diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-32.ll b/llvm/test/CodeGen/RISCV/rvp-simd-32.ll
index 7e4c7d4317eac..36a3430be3a1b 100644
--- a/llvm/test/CodeGen/RISCV/rvp-simd-32.ll
+++ b/llvm/test/CodeGen/RISCV/rvp-simd-32.ll
@@ -2461,3 +2461,68 @@ define <2 x i16> @test_pasa_x_h(<2 x i16> %a, <2 x i16> %b) {
%res = call <2 x i16> @llvm.riscv.pasa.v2i16(<2 x i16> %a, <2 x i16> %b)
ret <2 x i16> %res
}
+
+; Packed reduction sum
+define i32 @test_predsum_i8x4_i32(<4 x i8> %a, i32 %b) {
+; RV32-LABEL: test_predsum_i8x4_i32:
+; RV32: # %bb.0:
+; RV32-NEXT: predsum.bs a0, a0, a1
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_predsum_i8x4_i32:
+; RV64: # %bb.0:
+; RV64-NEXT: sext.w a1, a1
+; RV64-NEXT: zext.w a0, a0
+; RV64-NEXT: predsum.bs a0, a0, a1
+; RV64-NEXT: ret
+ %res = call i32 @llvm.riscv.predsum.i32.v4i8(<4 x i8> %a, i32 %b)
+ ret i32 %res
+}
+
+define i32 @test_predsumu_u8x4_u32(<4 x i8> %a, i32 %b) {
+; RV32-LABEL: test_predsumu_u8x4_u32:
+; RV32: # %bb.0:
+; RV32-NEXT: predsumu.bs a0, a0, a1
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_predsumu_u8x4_u32:
+; RV64: # %bb.0:
+; RV64-NEXT: zext.w a1, a1
+; RV64-NEXT: zext.w a0, a0
+; RV64-NEXT: predsumu.bs a0, a0, a1
+; RV64-NEXT: ret
+ %res = call i32 @llvm.riscv.predsumu.i32.v4i8(<4 x i8> %a, i32 %b)
+ ret i32 %res
+}
+
+define i32 @test_predsum_i16x2_i32(<2 x i16> %a, i32 %b) {
+; RV32-LABEL: test_predsum_i16x2_i32:
+; RV32: # %bb.0:
+; RV32-NEXT: predsum.hs a0, a0, a1
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_predsum_i16x2_i32:
+; RV64: # %bb.0:
+; RV64-NEXT: sext.w a1, a1
+; RV64-NEXT: zext.w a0, a0
+; RV64-NEXT: predsum.hs a0, a0, a1
+; RV64-NEXT: ret
+ %res = call i32 @llvm.riscv.predsum.i32.v2i16(<2 x i16> %a, i32 %b)
+ ret i32 %res
+}
+
+define i32 @test_predsumu_u16x2_u32(<2 x i16> %a, i32 %b) {
+; RV32-LABEL: test_predsumu_u16x2_u32:
+; RV32: # %bb.0:
+; RV32-NEXT: predsumu.hs a0, a0, a1
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_predsumu_u16x2_u32:
+; RV64: # %bb.0:
+; RV64-NEXT: zext.w a1, a1
+; RV64-NEXT: zext.w a0, a0
+; RV64-NEXT: predsumu.hs a0, a0, a1
+; RV64-NEXT: ret
+ %res = call i32 @llvm.riscv.predsumu.i32.v2i16(<2 x i16> %a, i32 %b)
+ ret i32 %res
+}
diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
index 808ff81102479..76838b44a9827 100644
--- a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
+++ b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
@@ -5332,3 +5332,158 @@ define <2 x i32> @test_pasa_x_w(<2 x i32> %a, <2 x i32> %b) {
%res = call <2 x i32> @llvm.riscv.pasa.v2i32(<2 x i32> %a, <2 x i32> %b)
ret <2 x i32> %res
}
+
+; Packed reduction sum
+define i32 @test_predsum_i8x8_i32(<8 x i8> %a, i32 %b) {
+; RV32-LABEL: test_predsum_i8x8_i32:
+; RV32: # %bb.0:
+; RV32-NEXT: predsum.dbs a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_predsum_i8x8_i32:
+; RV64: # %bb.0:
+; RV64-NEXT: sext.w a1, a1
+; RV64-NEXT: predsum.bs a0, a0, a1
+; RV64-NEXT: ret
+ %res = call i32 @llvm.riscv.predsum.i32.v8i8(<8 x i8> %a, i32 %b)
+ ret i32 %res
+}
+
+define i32 @test_predsumu_u8x8_u32(<8 x i8> %a, i32 %b) {
+; RV32-LABEL: test_predsumu_u8x8_u32:
+; RV32: # %bb.0:
+; RV32-NEXT: predsumu.dbs a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_predsumu_u8x8_u32:
+; RV64: # %bb.0:
+; RV64-NEXT: zext.w a1, a1
+; RV64-NEXT: predsumu.bs a0, a0, a1
+; RV64-NEXT: ret
+ %res = call i32 @llvm.riscv.predsumu.i32.v8i8(<8 x i8> %a, i32 %b)
+ ret i32 %res
+}
+
+define i64 @test_predsum_i8x8_i64(<8 x i8> %a, i64 %b) {
+; RV32-LABEL: test_predsum_i8x8_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: predsum.dbs a0, a0, zero
+; RV32-NEXT: wadda a2, a0, zero
+; RV32-NEXT: mvd a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_predsum_i8x8_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: predsum.bs a0, a0, a1
+; RV64-NEXT: ret
+ %res = call i64 @llvm.riscv.predsum.i64.v8i8(<8 x i8> %a, i64 %b)
+ ret i64 %res
+}
+
+define i64 @test_predsumu_u8x8_u64(<8 x i8> %a, i64 %b) {
+; RV32-LABEL: test_predsumu_u8x8_u64:
+; RV32: # %bb.0:
+; RV32-NEXT: predsumu.dbs a0, a0, zero
+; RV32-NEXT: waddau a2, a0, zero
+; RV32-NEXT: mvd a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_predsumu_u8x8_u64:
+; RV64: # %bb.0:
+; RV64-NEXT: predsumu.bs a0, a0, a1
+; RV64-NEXT: ret
+ %res = call i64 @llvm.riscv.predsumu.i64.v8i8(<8 x i8> %a, i64 %b)
+ ret i64 %res
+}
+
+define i32 @test_predsum_i16x4_i32(<4 x i16> %a, i32 %b) {
+; RV32-LABEL: test_predsum_i16x4_i32:
+; RV32: # %bb.0:
+; RV32-NEXT: predsum.dhs a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_predsum_i16x4_i32:
+; RV64: # %bb.0:
+; RV64-NEXT: sext.w a1, a1
+; RV64-NEXT: predsum.hs a0, a0, a1
+; RV64-NEXT: ret
+ %res = call i32 @llvm.riscv.predsum.i32.v4i16(<4 x i16> %a, i32 %b)
+ ret i32 %res
+}
+
+define i32 @test_predsumu_u16x4_u32(<4 x i16> %a, i32 %b) {
+; RV32-LABEL: test_predsumu_u16x4_u32:
+; RV32: # %bb.0:
+; RV32-NEXT: predsumu.dhs a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_predsumu_u16x4_u32:
+; RV64: # %bb.0:
+; RV64-NEXT: zext.w a1, a1
+; RV64-NEXT: predsumu.hs a0, a0, a1
+; RV64-NEXT: ret
+ %res = call i32 @llvm.riscv.predsumu.i32.v4i16(<4 x i16> %a, i32 %b)
+ ret i32 %res
+}
+
+define i64 @test_predsum_i16x4_i64(<4 x i16> %a, i64 %b) {
+; RV32-LABEL: test_predsum_i16x4_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: predsum.dhs a0, a0, zero
+; RV32-NEXT: wadda a2, a0, zero
+; RV32-NEXT: mvd a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_predsum_i16x4_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: predsum.hs a0, a0, a1
+; RV64-NEXT: ret
+ %res = call i64 @llvm.riscv.predsum.i64.v4i16(<4 x i16> %a, i64 %b)
+ ret i64 %res
+}
+
+define i64 @test_predsumu_u16x4_u64(<4 x i16> %a, i64 %b) {
+; RV32-LABEL: test_predsumu_u16x4_u64:
+; RV32: # %bb.0:
+; RV32-NEXT: predsumu.dhs a0, a0, zero
+; RV32-NEXT: waddau a2, a0, zero
+; RV32-NEXT: mvd a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_predsumu_u16x4_u64:
+; RV64: # %bb.0:
+; RV64-NEXT: predsumu.hs a0, a0, a1
+; RV64-NEXT: ret
+ %res = call i64 @llvm.riscv.predsumu.i64.v4i16(<4 x i16> %a, i64 %b)
+ ret i64 %res
+}
+
+define i64 @test_predsum_i32x2_i64(<2 x i32> %a, i64 %b) {
+; RV32-LABEL: test_predsum_i32x2_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: wadda a2, a0, a1
+; RV32-NEXT: mvd a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_predsum_i32x2_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: predsum.ws a0, a0, a1
+; RV64-NEXT: ret
+ %res = call i64 @llvm.riscv.predsum.i64.v2i32(<2 x i32> %a, i64 %b)
+ ret i64 %res
+}
+
+define i64 @test_predsumu_u32x2_u64(<2 x i32> %a, i64 %b) {
+; RV32-LABEL: test_predsumu_u32x2_u64:
+; RV32: # %bb.0:
+; RV32-NEXT: waddau a2, a0, a1
+; RV32-NEXT: mvd a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_predsumu_u32x2_u64:
+; RV64: # %bb.0:
+; RV64-NEXT: predsumu.ws a0, a0, a1
+; RV64-NEXT: ret
+ %res = call i64 @llvm.riscv.predsumu.i64.v2i32(<2 x i32> %a, i64 %b)
+ ret i64 %res
+}
More information about the llvm-commits
mailing list