[clang] [llvm] [RISCV][P-ext] Support Packed Multiplication with Widening Horizontal Addition and Accumulate intrinsics (PR #234450)
via cfe-commits
cfe-commits at lists.llvm.org
Wed Oct 21 19:13:24 PDT 2026
https://github.com/TelGome created https://github.com/llvm/llvm-project/pull/234450
Add LLVM intrinsics and codegen support for packed multiplication with widening horizontal addition and accumulate defined in the RISC-V P extension specification: https://github.com/riscv/riscv-p-spec/blob/master/P-ext-intrinsics.adoc#packed-multiplication-with-widening-horizontal-addition-and-accumulate
>From a4d57b8b5c791ffaa79704fc1b23f758f444f952 Mon Sep 17 00:00:00 2001
From: Dongyan Chen <chendongyan at isrc.iscas.ac.cn>
Date: Wed, 21 Oct 2026 08:49:47 +0000
Subject: [PATCH] [RISCV][P-ext] Support Packed Multiplication with Widening
Horizontal Addition and Accumulate intrinsics
---
clang/include/clang/Basic/BuiltinsRISCV.td | 8 ++
clang/lib/CodeGen/TargetBuiltins/RISCV.cpp | 34 +++++++
clang/lib/Headers/riscv_packed_simd.h | 8 ++
clang/test/CodeGen/RISCV/rvp-intrinsics.c | 72 ++++++++++++++
.../riscv_packed_simd.c | 52 ++++++++++
llvm/include/llvm/IR/IntrinsicsRISCV.td | 13 +++
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 89 ++++++++++++++++++
llvm/lib/Target/RISCV/RISCVInstrInfoP.td | 34 +++++++
llvm/test/CodeGen/RISCV/rvp-simd-64.ll | 94 +++++++++++++++++++
9 files changed, 404 insertions(+)
diff --git a/clang/include/clang/Basic/BuiltinsRISCV.td b/clang/include/clang/Basic/BuiltinsRISCV.td
index b7305e46cad6dc..f2e16692876ce9 100644
--- a/clang/include/clang/Basic/BuiltinsRISCV.td
+++ b/clang/include/clang/Basic/BuiltinsRISCV.td
@@ -404,6 +404,14 @@ def pm2wsub_i64 : RISCVBuiltin<"int64_t(_Vector<2, short>, _Vector<2, short>)">;
def pm2wsub_x_i64 : RISCVBuiltin<"int64_t(_Vector<2, short>, _Vector<2, short>)">;
def pm2waddsu_i64 : RISCVBuiltin<"int64_t(_Vector<2, short>, _Vector<2, unsigned short>)">;
+// Packed Multiplication with Widening Horizontal Addition and Accumulate (32-bit)
+def pm2wadda_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, short>, _Vector<2, short>)">;
+def pm2wadda_x_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, short>, _Vector<2, short>)">;
+def pm2waddau_u64 : RISCVBuiltin<"uint64_t(uint64_t, _Vector<2, unsigned short>, _Vector<2, unsigned short>)">;
+def pm2wsuba_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, short>, _Vector<2, short>)">;
+def pm2wsuba_x_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, short>, _Vector<2, short>)">;
+def pm2waddasu_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, short>, _Vector<2, unsigned short>)">;
+
// Packed Absolute Difference Sum (32-bit)
def pabdsumu_u8x4_u32 : RISCVBuiltin<"unsigned int(_Vector<4, unsigned char>, _Vector<4, unsigned char>)">;
def pabdsumau_u8x4_u32 : RISCVBuiltin<"unsigned int(unsigned int, _Vector<4, unsigned char>, _Vector<4, unsigned char>)">;
diff --git a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
index 879693b9811c39..ebbba76bcee6a3 100644
--- a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
@@ -1891,6 +1891,40 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID,
break;
}
+ // Packed Multiplication with Widening Horizontal Addition and Accumulate.
+ case RISCV::BI__builtin_riscv_pm2wadda_i64:
+ case RISCV::BI__builtin_riscv_pm2wadda_x_i64:
+ case RISCV::BI__builtin_riscv_pm2waddau_u64:
+ case RISCV::BI__builtin_riscv_pm2wsuba_i64:
+ case RISCV::BI__builtin_riscv_pm2wsuba_x_i64:
+ case RISCV::BI__builtin_riscv_pm2waddasu_i64: {
+ switch (BuiltinID) {
+ default:
+ llvm_unreachable("unexpected builtin ID");
+ case RISCV::BI__builtin_riscv_pm2wadda_i64:
+ ID = Intrinsic::riscv_pm2wadda;
+ break;
+ case RISCV::BI__builtin_riscv_pm2wadda_x_i64:
+ ID = Intrinsic::riscv_pm2wadda_x;
+ break;
+ case RISCV::BI__builtin_riscv_pm2waddau_u64:
+ ID = Intrinsic::riscv_pm2waddau;
+ break;
+ case RISCV::BI__builtin_riscv_pm2wsuba_i64:
+ ID = Intrinsic::riscv_pm2wsuba;
+ break;
+ case RISCV::BI__builtin_riscv_pm2wsuba_x_i64:
+ ID = Intrinsic::riscv_pm2wsuba_x;
+ break;
+ case RISCV::BI__builtin_riscv_pm2waddasu_i64:
+ ID = Intrinsic::riscv_pm2waddasu;
+ break;
+ }
+
+ IntrinsicTypes = {ResultType, Ops[1]->getType()};
+ break;
+ }
+
// Packed Reduction Sum
case RISCV::BI__builtin_riscv_predsum_i8x4_i32:
case RISCV::BI__builtin_riscv_predsum_i16x2_i32:
diff --git a/clang/lib/Headers/riscv_packed_simd.h b/clang/lib/Headers/riscv_packed_simd.h
index a4b27eb2e62190..13c8ae8c3fdba8 100644
--- a/clang/lib/Headers/riscv_packed_simd.h
+++ b/clang/lib/Headers/riscv_packed_simd.h
@@ -1193,6 +1193,14 @@ __packed_binary_builtin_mixed(pm2wsub_i64, int64_t, int16x2_t, int16x2_t, __buil
__packed_binary_builtin_mixed(pm2wsub_x_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wsub_x_i64)
__packed_binary_builtin_mixed(pm2waddsu_i64, int64_t, int16x2_t, uint16x2_t, __builtin_riscv_pm2waddsu_i64)
+/* Packed Multiplication with Widening Horizontal Addition and Accumulate */
+__packed_ternary_builtin_mixed(pm2wadda_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wadda_i64)
+__packed_ternary_builtin_mixed(pm2wadda_x_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wadda_x_i64)
+__packed_ternary_builtin_mixed(pm2waddau_u64, uint64_t, uint16x2_t, uint16x2_t, __builtin_riscv_pm2waddau_u64)
+__packed_ternary_builtin_mixed(pm2wsuba_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wsuba_i64)
+__packed_ternary_builtin_mixed(pm2wsuba_x_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wsuba_x_i64)
+__packed_ternary_builtin_mixed(pm2waddasu_i64, int64_t, int16x2_t, uint16x2_t, __builtin_riscv_pm2waddasu_i64)
+
/* Packed Absolute Difference Sum (32-bit) */
__packed_abdsum(pabdsumu_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumu_u8x4_u32)
__packed_ternary_builtin_cast(pabdsumau_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumau_u8x4_u32)
diff --git a/clang/test/CodeGen/RISCV/rvp-intrinsics.c b/clang/test/CodeGen/RISCV/rvp-intrinsics.c
index bc9d12a8f59f80..56007cf747581c 100644
--- a/clang/test/CodeGen/RISCV/rvp-intrinsics.c
+++ b/clang/test/CodeGen/RISCV/rvp-intrinsics.c
@@ -10096,3 +10096,75 @@ int64_t test_pm2wsub_x_i64(int16x2_t rs1, int16x2_t rs2) {
int64_t test_pm2waddsu_i64(int16x2_t rs1, uint16x2_t rs2) {
return __riscv_pm2waddsu_i64(rs1, rs2);
}
+
+// CHECK-LABEL: define dso_local i64 @test_pm2wadda_i64(
+// CHECK-SAME: i64 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2wadda.i64.v2i16(i64 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// CHECK-NEXT: ret i64 [[TMP2]]
+//
+int64_t test_pm2wadda_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2wadda_i64(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pm2wadda_x_i64(
+// CHECK-SAME: i64 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2wadda.x.i64.v2i16(i64 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// CHECK-NEXT: ret i64 [[TMP2]]
+//
+int64_t test_pm2wadda_x_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2wadda_x_i64(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pm2waddau_u64(
+// CHECK-SAME: i64 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2waddau.i64.v2i16(i64 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// CHECK-NEXT: ret i64 [[TMP2]]
+//
+uint64_t test_pm2waddau_u64(uint64_t rd, uint16x2_t rs1, uint16x2_t rs2) {
+ return __riscv_pm2waddau_u64(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pm2wsuba_i64(
+// CHECK-SAME: i64 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2wsuba.i64.v2i16(i64 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// CHECK-NEXT: ret i64 [[TMP2]]
+//
+int64_t test_pm2wsuba_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2wsuba_i64(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pm2wsuba_x_i64(
+// CHECK-SAME: i64 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2wsuba.x.i64.v2i16(i64 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// CHECK-NEXT: ret i64 [[TMP2]]
+//
+int64_t test_pm2wsuba_x_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2wsuba_x_i64(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pm2waddasu_i64(
+// CHECK-SAME: i64 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2waddasu.i64.v2i16(i64 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// CHECK-NEXT: ret i64 [[TMP2]]
+//
+int64_t test_pm2waddasu_i64(int64_t rd, int16x2_t rs1, uint16x2_t rs2) {
+ return __riscv_pm2waddasu_i64(rd, rs1, rs2);
+}
diff --git a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
index a54807804b9836..9f784bae2c8035 100644
--- a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
+++ b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
@@ -4560,6 +4560,58 @@ int64_t test_pm2waddsu_i64(int16x2_t rs1, uint16x2_t rs2) {
return __riscv_pm2waddsu_i64(rs1, rs2);
}
+// Packed Multiplication with Widening Horizontal Addition and Accumulate
+// CHECK-LABEL: test_pm2wadda_i64:
+// RV32: pm2wadda.h
+// RV64: pack{{[[:space:]]}}
+// RV64: pm4adda.h
+int64_t test_pm2wadda_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2wadda_i64(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2wadda_x_i64:
+// RV32: pm2wadda.hx
+// RV64: pack{{[[:space:]]}}
+// RV64: ppairoe.h
+// RV64: pm4adda.h
+int64_t test_pm2wadda_x_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2wadda_x_i64(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2waddau_u64:
+// RV32: pm2waddau.h
+// RV64: pack{{[[:space:]]}}
+// RV64: pm4addau.h
+uint64_t test_pm2waddau_u64(uint64_t rd, uint16x2_t rs1, uint16x2_t rs2) {
+ return __riscv_pm2waddau_u64(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2wsuba_i64:
+// RV32: pm2wsuba.h
+// RV64: pm2sub.h
+// RV64: zext.w
+// RV64: predsum.ws
+int64_t test_pm2wsuba_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2wsuba_i64(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2wsuba_x_i64:
+// RV32: pm2wsuba.hx
+// RV64: pm2sub.hx
+// RV64: zext.w
+// RV64: predsum.ws
+int64_t test_pm2wsuba_x_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2wsuba_x_i64(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2waddasu_i64:
+// RV32: pm2waddasu.h
+// RV64: pack{{[[:space:]]}}
+// RV64: pm4addasu.h
+int64_t test_pm2waddasu_i64(int64_t rd, int16x2_t rs1, uint16x2_t rs2) {
+ return __riscv_pm2waddasu_i64(rd, rs1, rs2);
+}
+
// Packed Multiply Parts.
// CHECK-LABEL: test_pmul_b00_i16x2:
// RV32: pmul.h.b00
diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td
index 321d6ac5e93af0..b5fb800309ea4b 100644
--- a/llvm/include/llvm/IR/IntrinsicsRISCV.td
+++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td
@@ -2319,6 +2319,19 @@ class RVPBinaryIntrinsic
def int_riscv_pm2wsub_x : RVPHorizontalMulIntrinsic;
def int_riscv_pm2waddsu : RVPHorizontalMulIntrinsic;
+ // Packed Multiplication with Widening Horizontal Addition and Accumulate.
+ class RVPHorizontalMulAccIntrinsic
+ : DefaultAttrsIntrinsic<[llvm_any_ty],
+ [LLVMMatchType<0>, llvm_anyvector_ty,
+ LLVMMatchType<1>],
+ [IntrNoMem, IntrSpeculatable]>;
+ def int_riscv_pm2wadda : RVPHorizontalMulAccIntrinsic;
+ def int_riscv_pm2wadda_x : RVPHorizontalMulAccIntrinsic;
+ def int_riscv_pm2waddau : RVPHorizontalMulAccIntrinsic;
+ def int_riscv_pm2wsuba : RVPHorizontalMulAccIntrinsic;
+ def int_riscv_pm2wsuba_x : RVPHorizontalMulAccIntrinsic;
+ def int_riscv_pm2waddasu : RVPHorizontalMulAccIntrinsic;
+
// Packed Multiply Parts. The packed forms multiply the selected element of
// every pair, so the operands are the result with each element split in two.
class RVPPackedMulPartsIntrinsic
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 0cc05e5f2c374d..e6f01e3c3d636a 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -12647,6 +12647,26 @@ static unsigned getRVPHorizontalMulOpcode(unsigned IntNo) {
}
}
+static unsigned getRVPHorizontalMulAccOpcode(unsigned IntNo) {
+ switch (IntNo) {
+ default:
+ llvm_unreachable(
+ "Unexpected RISC-V packed horizontal multiply-accumulate intrinsic");
+ case Intrinsic::riscv_pm2wadda:
+ return RISCVISD::PM2WADDA;
+ case Intrinsic::riscv_pm2wadda_x:
+ return RISCVISD::PM2WADDA_X;
+ case Intrinsic::riscv_pm2waddau:
+ return RISCVISD::PM2WADDAU;
+ case Intrinsic::riscv_pm2wsuba:
+ return RISCVISD::PM2WSUBA;
+ case Intrinsic::riscv_pm2wsuba_x:
+ return RISCVISD::PM2WSUBA_X;
+ case Intrinsic::riscv_pm2waddasu:
+ return RISCVISD::PM2WADDASU;
+ }
+}
+
static SDValue lowerRV32HorizontalMul64(unsigned IntNo, SDValue Rs1,
SDValue Rs2, const SDLoc &DL,
SelectionDAG &DAG) {
@@ -13706,6 +13726,55 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
: RISCVISD::PM4ADD;
return DAG.getNode(MulOpc, DL, MVT::i64, A, B);
}
+ case Intrinsic::riscv_pm2wadda:
+ case Intrinsic::riscv_pm2wadda_x:
+ case Intrinsic::riscv_pm2waddau:
+ case Intrinsic::riscv_pm2wsuba:
+ case Intrinsic::riscv_pm2wsuba_x:
+ case Intrinsic::riscv_pm2waddasu: {
+
+ assert(Subtarget.is64Bit() && Op.getValueType() == MVT::i64 &&
+ "Unexpected widening horizontal multiply-accumulate legalization");
+ SDValue Rd = Op.getOperand(1);
+ SDValue Rs1 = Op.getOperand(2);
+ SDValue Rs2 = Op.getOperand(3);
+
+ if (IntNo == Intrinsic::riscv_pm2wsuba ||
+ IntNo == Intrinsic::riscv_pm2wsuba_x) {
+
+ unsigned SubIntNo = IntNo == Intrinsic::riscv_pm2wsuba
+ ? Intrinsic::riscv_pm2sub
+ : Intrinsic::riscv_pm2sub_x;
+ SDValue SubId = DAG.getTargetConstant(SubIntNo, DL, MVT::i64);
+ SDValue Diff =
+ DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32, SubId, Rs1, Rs2);
+ SDValue DiffWide = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Diff);
+ SDValue Packed = DAG.getBitcast(MVT::v2i32, DiffWide);
+ SDValue SumId =
+ DAG.getTargetConstant(Intrinsic::riscv_predsum, DL, MVT::i64);
+ return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64, SumId, Packed,
+ Rd);
+ }
+
+ auto WidenAnyAcc = [&](SDValue V) {
+ return DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16, V,
+ DAG.getUNDEF(MVT::v2i16));
+ };
+ SDValue A = WidenAnyAcc(Rs1);
+ SDValue B;
+ if (IntNo == Intrinsic::riscv_pm2wadda_x) {
+ SDValue Rs2Wide = widenPackedVectorWithZeros(DAG, DL, Rs2, MVT::v4i16);
+ B = DAG.getNode(RISCVISD::PPAIROE, DL, MVT::v4i16, Rs2Wide, Rs2Wide);
+ } else {
+ B = widenPackedVectorWithZeros(DAG, DL, Rs2, MVT::v4i16);
+ }
+
+ unsigned AccOpc = IntNo == Intrinsic::riscv_pm2waddau ? RISCVISD::PM4ADDAU
+ : IntNo == Intrinsic::riscv_pm2waddasu
+ ? RISCVISD::PM4ADDASU
+ : RISCVISD::PM4ADDA;
+ return DAG.getNode(AccOpc, DL, MVT::i64, Rd, A, B);
+ }
case Intrinsic::riscv_pmerge: {
EVT VT = Op.getValueType();
auto buildMerge = [&](SDValue Rs1, SDValue Rs2, SDValue Mask,
@@ -17926,6 +17995,26 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
return;
}
+ case Intrinsic::riscv_pm2wadda:
+ case Intrinsic::riscv_pm2wadda_x:
+ case Intrinsic::riscv_pm2waddau:
+ case Intrinsic::riscv_pm2wsuba:
+ case Intrinsic::riscv_pm2wsuba_x:
+ case Intrinsic::riscv_pm2waddasu: {
+
+ assert(!Subtarget.is64Bit() && N->getSimpleValueType(0) == MVT::i64 &&
+ "Unexpected widening horizontal multiply-accumulate legalization");
+ SDValue Rs1 = N->getOperand(2);
+ SDValue Rs2 = N->getOperand(3);
+ auto [RdLo, RdHi] =
+ DAG.SplitScalar(N->getOperand(1), DL, MVT::i32, MVT::i32);
+ SDValue Rd = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2i32, RdLo, RdHi);
+ SDValue Res = DAG.getNode(getRVPHorizontalMulAccOpcode(IntNo), DL,
+ MVT::v2i32, Rd, Rs1, Rs2);
+ Results.push_back(DAG.getBitcast(MVT::i64, Res));
+ return;
+ }
+
case Intrinsic::riscv_pmul_00:
case Intrinsic::riscv_pmul_01:
case Intrinsic::riscv_pmul_11:
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
index b344638a754c88..d1e5782f0cb6eb 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
@@ -1942,8 +1942,14 @@ def riscv_pmqrwacc_h
: RVSDNode<"PMQRWACC_H", SDT_RISCVPackedWideningAcc>;
def riscv_pm2wadda
: RVSDNode<"PM2WADDA", SDT_RISCVPackedWideningAcc>;
+def riscv_pm2wadda_x
+ : RVSDNode<"PM2WADDA_X", SDT_RISCVPackedWideningAcc>;
def riscv_pm2waddau
: RVSDNode<"PM2WADDAU", SDT_RISCVPackedWideningAcc>;
+def riscv_pm2wsuba
+ : RVSDNode<"PM2WSUBA", SDT_RISCVPackedWideningAcc>;
+def riscv_pm2wsuba_x
+ : RVSDNode<"PM2WSUBA_X", SDT_RISCVPackedWideningAcc>;
def riscv_pm2waddasu
: RVSDNode<"PM2WADDASU", SDT_RISCVPackedWideningAcc>;
@@ -2064,6 +2070,16 @@ def riscv_pm2sub_x : RVSDNode<"PM2SUB_X", SDT_RISCVHorizontalMul>;
def riscv_pm4addsu : RVSDNode<"PM4ADDSU", SDT_RISCVHorizontalMul>;
def riscv_pm2addsu : RVSDNode<"PM2ADDSU", SDT_RISCVHorizontalMul>;
+// Horizontal-multiply-and-accumulate internal helpers, used only to
+// synthesize the public pm2wadda(_x)/au/asu intrinsics on RV64 (never
+// exposed as IR intrinsics themselves).
+def SDT_RISCVHorizontalMulAcc
+ : SDTypeProfile<1, 3, [SDTCisSameAs<0, 1>, SDTCisVec<2>,
+ SDTCisSameAs<2, 3>]>;
+def riscv_pm4adda : RVSDNode<"PM4ADDA", SDT_RISCVHorizontalMulAcc>;
+def riscv_pm4addau : RVSDNode<"PM4ADDAU", SDT_RISCVHorizontalMulAcc>;
+def riscv_pm4addasu : RVSDNode<"PM4ADDASU", SDT_RISCVHorizontalMulAcc>;
+
// "Q-format" multiplication
def riscv_mulq : RVSDNode<"MULQ", SDTIntBinOp, [SDNPCommutative]>;
def riscv_mulqr : RVSDNode<"MULQR", SDTIntBinOp, [SDNPCommutative]>;
@@ -2208,6 +2224,12 @@ class PatHorizontalMul<SDPatternOperator OpNode, RVInst Inst,
: Pat<(ResultVT (OpNode (SrcVT GPR:$rs1), (SrcVT GPR:$rs2))),
(Inst GPR:$rs1, GPR:$rs2)>;
+class PatHorizontalMulAcc<SDPatternOperator OpNode, RVInst Inst,
+ ValueType ResultVT, ValueType SrcVT>
+ : Pat<(ResultVT (OpNode (ResultVT GPR:$rd), (SrcVT GPR:$rs1),
+ (SrcVT GPR:$rs2))),
+ (Inst GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+
// (rs1 << 1) + rs2
class PatPSh1Add<RVInst Inst, ValueType vt>
: Pat<(vt (add (riscv_psll (vt GPR:$rs1), (XLenVT 1)), (vt GPR:$rs2))),
@@ -2754,9 +2776,18 @@ let append Predicates = [IsRV32] in {
def : Pat<(v2i32 (riscv_pm2wadda (v2i32 GPRPair:$rd),
(v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
(PM2WADDA_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(v2i32 (riscv_pm2wadda_x (v2i32 GPRPair:$rd),
+ (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
+ (PM2WADDA_HX GPRPair:$rd, GPR:$rs1, GPR:$rs2)>;
def : Pat<(v2i32 (riscv_pm2waddau (v2i32 GPRPair:$rd),
(v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
(PM2WADDAU_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(v2i32 (riscv_pm2wsuba (v2i32 GPRPair:$rd),
+ (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
+ (PM2WSUBA_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(v2i32 (riscv_pm2wsuba_x (v2i32 GPRPair:$rd),
+ (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
+ (PM2WSUBA_HX GPRPair:$rd, GPR:$rs1, GPR:$rs2)>;
def : Pat<(v2i32 (riscv_pm2waddasu (v2i32 GPRPair:$rd),
(v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
(PM2WADDASU_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>;
@@ -3452,6 +3483,9 @@ let append Predicates = [IsRV64] in {
def : PatHorizontalMul<riscv_pm4add, PM4ADD_H, i64, v4i16>;
def : PatHorizontalMul<riscv_pm4addu, PM4ADDU_H, i64, v4i16>;
def : PatHorizontalMul<riscv_pm4addsu, PM4ADDSU_H, i64, v4i16>;
+ def : PatHorizontalMulAcc<riscv_pm4adda, PM4ADDA_H, i64, v4i16>;
+ def : PatHorizontalMulAcc<riscv_pm4addau, PM4ADDAU_H, i64, v4i16>;
+ def : PatHorizontalMulAcc<riscv_pm4addasu, PM4ADDASU_H, i64, v4i16>;
// 8/16/32-bit multiply low patterns
// FIXME custom lower
diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
index 5b17c562dce16d..150be5f940fa38 100644
--- a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
+++ b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
@@ -8089,6 +8089,100 @@ define i64 @test_pm2waddsu_v2i16_i64(<2 x i16> %a, <2 x i16> %b) {
ret i64 %r
}
+; Packed Multiplication with Widening Horizontal Addition and Accumulate.
+define i64 @test_pm2wadda_v2i16_i64(i64 %rd, <2 x i16> %a, <2 x i16> %b) {
+; RV32-LABEL: test_pm2wadda_v2i16_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2wadda.h a0, a2, a3
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2wadda_v2i16_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pack a2, a2, zero
+; RV64-NEXT: pm4adda.h a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2wadda.i64.v2i16(i64 %rd, <2 x i16> %a, <2 x i16> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm2wadda_x_v2i16_i64(i64 %rd, <2 x i16> %a, <2 x i16> %b) {
+; RV32-LABEL: test_pm2wadda_x_v2i16_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2wadda.hx a0, a2, a3
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2wadda_x_v2i16_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pack a2, a2, zero
+; RV64-NEXT: ppairoe.h a2, a2, a2
+; RV64-NEXT: pm4adda.h a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2wadda.x.i64.v2i16(i64 %rd, <2 x i16> %a, <2 x i16> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm2waddau_v2i16_i64(i64 %rd, <2 x i16> %a, <2 x i16> %b) {
+; RV32-LABEL: test_pm2waddau_v2i16_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2waddau.h a0, a2, a3
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2waddau_v2i16_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pack a2, a2, zero
+; RV64-NEXT: pm4addau.h a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2waddau.i64.v2i16(i64 %rd, <2 x i16> %a, <2 x i16> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm2wsuba_v2i16_i64(i64 %rd, <2 x i16> %a, <2 x i16> %b) {
+; RV32-LABEL: test_pm2wsuba_v2i16_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2wsuba.h a0, a2, a3
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2wsuba_v2i16_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2sub.h a1, a1, a2
+; RV64-NEXT: zext.w a1, a1
+; RV64-NEXT: predsum.ws a0, a1, a0
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2wsuba.i64.v2i16(i64 %rd, <2 x i16> %a, <2 x i16> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm2wsuba_x_v2i16_i64(i64 %rd, <2 x i16> %a, <2 x i16> %b) {
+; RV32-LABEL: test_pm2wsuba_x_v2i16_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2wsuba.hx a0, a2, a3
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2wsuba_x_v2i16_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2sub.hx a1, a1, a2
+; RV64-NEXT: zext.w a1, a1
+; RV64-NEXT: predsum.ws a0, a1, a0
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2wsuba.x.i64.v2i16(i64 %rd, <2 x i16> %a, <2 x i16> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm2waddasu_v2i16_i64(i64 %rd, <2 x i16> %a, <2 x i16> %b) {
+; RV32-LABEL: test_pm2waddasu_v2i16_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2waddasu.h a0, a2, a3
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2waddasu_v2i16_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pack a2, a2, zero
+; RV64-NEXT: pm4addasu.h a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2waddasu.i64.v2i16(i64 %rd, <2 x i16> %a, <2 x i16> %b)
+ ret i64 %r
+}
+
; Packed Multiply Parts Accumulate.
declare <2 x i32> @llvm.riscv.pmacc.00.v2i32(<2 x i32>, <4 x i16>, <4 x i16>)
declare <2 x i32> @llvm.riscv.pmacc.01.v2i32(<2 x i32>, <4 x i16>, <4 x i16>)
More information about the cfe-commits
mailing list