[clang] [llvm] [RISCV] Support Packed Multiplication with Horizontal Addition (PR #218430)
Hongyu Chen via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 26 11:46:25 PDT 2026
https://github.com/XChy updated https://github.com/llvm/llvm-project/pull/218430
>From cb9d1c63130ad55869409db3d225323ed38afb4d Mon Sep 17 00:00:00 2001
From: XChy <xxs_chy at outlook.com>
Date: Mon, 24 Aug 2026 22:10:31 +0800
Subject: [PATCH 1/4] [RISCV] Support Packed Multiplication with Horizontal
Addition
---
clang/include/clang/Basic/BuiltinsRISCV.td | 41 ++
clang/lib/CodeGen/TargetBuiltins/RISCV.cpp | 110 +++++
clang/lib/Headers/riscv_packed_simd.h | 41 ++
.../riscv_packed_simd.c | 285 +++++++++++++
llvm/include/llvm/IR/IntrinsicsRISCV.td | 19 +
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 100 +++++
llvm/lib/Target/RISCV/RISCVInstrInfoP.td | 140 +++++++
llvm/test/CodeGen/RISCV/rvp-simd-32.ll | 117 ++++++
llvm/test/CodeGen/RISCV/rvp-simd-64.ll | 375 ++++++++++++++++++
9 files changed, 1228 insertions(+)
diff --git a/clang/include/clang/Basic/BuiltinsRISCV.td b/clang/include/clang/Basic/BuiltinsRISCV.td
index b292ffc0140e8..4f648f4fbd873 100644
--- a/clang/include/clang/Basic/BuiltinsRISCV.td
+++ b/clang/include/clang/Basic/BuiltinsRISCV.td
@@ -291,6 +291,47 @@ def pmhaccsu_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<2, i
def pmhraccsu_i16x4 : RISCVBuiltin<"_Vector<4, short>(_Vector<4, short>, _Vector<4, short>, _Vector<4, unsigned short>)">;
def pmhraccsu_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<2, int>, _Vector<2, unsigned int>)">;
+// Packed Multiplication with Horizontal Addition (32-bit)
+def pm4add_i8x4 : RISCVBuiltin<"int(_Vector<4, signed char>, _Vector<4, signed char>)">;
+def pm2add_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">;
+def pm2add_x_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">;
+def pm4addu_u8x4 : RISCVBuiltin<"unsigned int(_Vector<4, unsigned char>, _Vector<4, unsigned char>)">;
+def pm2addu_u16x2 : RISCVBuiltin<"unsigned int(_Vector<2, unsigned short>, _Vector<2, unsigned short>)">;
+def pmq2add_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">;
+def pmqr2add_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">;
+def pm2sadd_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">;
+def pm2sadd_x_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">;
+def pm2sub_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">;
+def pm2sub_x_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">;
+def pm4addsu_i8x4 : RISCVBuiltin<"int(_Vector<4, signed char>, _Vector<4, unsigned char>)">;
+def pm2addsu_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, unsigned short>)">;
+
+// Packed Multiplication with Horizontal Addition (64-bit)
+def pm4add_i8x8 : RISCVBuiltin<"_Vector<2, int>(_Vector<8, signed char>, _Vector<8, signed char>)">;
+def pm2add_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">;
+def pm2add_x_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">;
+def pm4addu_u8x8 : RISCVBuiltin<"_Vector<2, unsigned int>(_Vector<8, unsigned char>, _Vector<8, unsigned char>)">;
+def pm2addu_u16x4 : RISCVBuiltin<"_Vector<2, unsigned int>(_Vector<4, unsigned short>, _Vector<4, unsigned short>)">;
+def pmq2add_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">;
+def pmqr2add_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">;
+def pm2sadd_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">;
+def pm2sadd_x_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">;
+def pm2sub_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">;
+def pm2sub_x_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">;
+def pm4addsu_i8x8 : RISCVBuiltin<"_Vector<2, int>(_Vector<8, signed char>, _Vector<8, unsigned char>)">;
+def pm2addsu_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, unsigned short>)">;
+def pm2add_i32x2 : RISCVBuiltin<"int64_t(_Vector<2, int>, _Vector<2, int>)">;
+def pm2add_x_i32x2 : RISCVBuiltin<"int64_t(_Vector<2, int>, _Vector<2, int>)">;
+def pm2addu_u32x2 : RISCVBuiltin<"uint64_t(_Vector<2, unsigned int>, _Vector<2, unsigned int>)">;
+def pmq2add_i32x2 : RISCVBuiltin<"int64_t(_Vector<2, int>, _Vector<2, int>)">;
+def pm2sub_i32x2 : RISCVBuiltin<"int64_t(_Vector<2, int>, _Vector<2, int>)">;
+def pm2sub_x_i32x2 : RISCVBuiltin<"int64_t(_Vector<2, int>, _Vector<2, int>)">;
+def pm2addsu_i32x2 : RISCVBuiltin<"int64_t(_Vector<2, int>, _Vector<2, unsigned int>)">;
+def pmqr2add_i32x2 : RISCVBuiltin<"int64_t(_Vector<2, int>, _Vector<2, int>)">;
+def pm4add_i16x4 : RISCVBuiltin<"int64_t(_Vector<4, short>, _Vector<4, short>)">;
+def pm4addu_u16x4 : RISCVBuiltin<"uint64_t(_Vector<4, unsigned short>, _Vector<4, unsigned short>)">;
+def pm4addsu_i16x4 : RISCVBuiltin<"int64_t(_Vector<4, short>, _Vector<4, unsigned short>)">;
+
// Packed Absolute Difference Sum (32-bit)
def pabdsumu_u8x4_u32 : RISCVBuiltin<"unsigned int(_Vector<4, unsigned char>, _Vector<4, unsigned char>)">;
def pabdsumau_u8x4_u32 : RISCVBuiltin<"unsigned int(unsigned int, _Vector<4, unsigned char>, _Vector<4, unsigned char>)">;
diff --git a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
index 91f451e34c2c9..eee029a513d03 100644
--- a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
@@ -1531,6 +1531,116 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID,
break;
}
+ // Packed Multiplication with Horizontal Addition
+ case RISCV::BI__builtin_riscv_pm4add_i8x4:
+ case RISCV::BI__builtin_riscv_pm4add_i8x8:
+ case RISCV::BI__builtin_riscv_pm4add_i16x4:
+ case RISCV::BI__builtin_riscv_pm2add_i16x2:
+ case RISCV::BI__builtin_riscv_pm2add_i16x4:
+ case RISCV::BI__builtin_riscv_pm2add_i32x2:
+ case RISCV::BI__builtin_riscv_pm2add_x_i16x2:
+ case RISCV::BI__builtin_riscv_pm2add_x_i16x4:
+ case RISCV::BI__builtin_riscv_pm2add_x_i32x2:
+ case RISCV::BI__builtin_riscv_pm4addu_u8x4:
+ case RISCV::BI__builtin_riscv_pm4addu_u8x8:
+ case RISCV::BI__builtin_riscv_pm4addu_u16x4:
+ case RISCV::BI__builtin_riscv_pm2addu_u16x2:
+ case RISCV::BI__builtin_riscv_pm2addu_u16x4:
+ case RISCV::BI__builtin_riscv_pm2addu_u32x2:
+ case RISCV::BI__builtin_riscv_pmq2add_i16x2:
+ case RISCV::BI__builtin_riscv_pmq2add_i16x4:
+ case RISCV::BI__builtin_riscv_pmq2add_i32x2:
+ case RISCV::BI__builtin_riscv_pmqr2add_i16x2:
+ case RISCV::BI__builtin_riscv_pmqr2add_i16x4:
+ case RISCV::BI__builtin_riscv_pmqr2add_i32x2:
+ case RISCV::BI__builtin_riscv_pm2sadd_i16x2:
+ case RISCV::BI__builtin_riscv_pm2sadd_i16x4:
+ case RISCV::BI__builtin_riscv_pm2sadd_x_i16x2:
+ case RISCV::BI__builtin_riscv_pm2sadd_x_i16x4:
+ case RISCV::BI__builtin_riscv_pm2sub_i16x2:
+ case RISCV::BI__builtin_riscv_pm2sub_i16x4:
+ case RISCV::BI__builtin_riscv_pm2sub_i32x2:
+ case RISCV::BI__builtin_riscv_pm2sub_x_i16x2:
+ case RISCV::BI__builtin_riscv_pm2sub_x_i16x4:
+ case RISCV::BI__builtin_riscv_pm2sub_x_i32x2:
+ case RISCV::BI__builtin_riscv_pm4addsu_i8x4:
+ case RISCV::BI__builtin_riscv_pm4addsu_i8x8:
+ case RISCV::BI__builtin_riscv_pm4addsu_i16x4:
+ case RISCV::BI__builtin_riscv_pm2addsu_i16x2:
+ case RISCV::BI__builtin_riscv_pm2addsu_i16x4:
+ case RISCV::BI__builtin_riscv_pm2addsu_i32x2: {
+ switch (BuiltinID) {
+ default:
+ llvm_unreachable("unexpected builtin ID");
+ case RISCV::BI__builtin_riscv_pm4add_i8x4:
+ case RISCV::BI__builtin_riscv_pm4add_i8x8:
+ case RISCV::BI__builtin_riscv_pm4add_i16x4:
+ ID = Intrinsic::riscv_pm4add;
+ break;
+ case RISCV::BI__builtin_riscv_pm2add_i16x2:
+ case RISCV::BI__builtin_riscv_pm2add_i16x4:
+ case RISCV::BI__builtin_riscv_pm2add_i32x2:
+ ID = Intrinsic::riscv_pm2add;
+ break;
+ case RISCV::BI__builtin_riscv_pm2add_x_i16x2:
+ case RISCV::BI__builtin_riscv_pm2add_x_i16x4:
+ case RISCV::BI__builtin_riscv_pm2add_x_i32x2:
+ ID = Intrinsic::riscv_pm2add_x;
+ break;
+ case RISCV::BI__builtin_riscv_pm4addu_u8x4:
+ case RISCV::BI__builtin_riscv_pm4addu_u8x8:
+ case RISCV::BI__builtin_riscv_pm4addu_u16x4:
+ ID = Intrinsic::riscv_pm4addu;
+ break;
+ case RISCV::BI__builtin_riscv_pm2addu_u16x2:
+ case RISCV::BI__builtin_riscv_pm2addu_u16x4:
+ case RISCV::BI__builtin_riscv_pm2addu_u32x2:
+ ID = Intrinsic::riscv_pm2addu;
+ break;
+ case RISCV::BI__builtin_riscv_pmq2add_i16x2:
+ case RISCV::BI__builtin_riscv_pmq2add_i16x4:
+ case RISCV::BI__builtin_riscv_pmq2add_i32x2:
+ ID = Intrinsic::riscv_pmq2add;
+ break;
+ case RISCV::BI__builtin_riscv_pmqr2add_i16x2:
+ case RISCV::BI__builtin_riscv_pmqr2add_i16x4:
+ case RISCV::BI__builtin_riscv_pmqr2add_i32x2:
+ ID = Intrinsic::riscv_pmqr2add;
+ break;
+ case RISCV::BI__builtin_riscv_pm2sadd_i16x2:
+ case RISCV::BI__builtin_riscv_pm2sadd_i16x4:
+ ID = Intrinsic::riscv_pm2sadd;
+ break;
+ case RISCV::BI__builtin_riscv_pm2sadd_x_i16x2:
+ case RISCV::BI__builtin_riscv_pm2sadd_x_i16x4:
+ ID = Intrinsic::riscv_pm2sadd_x;
+ break;
+ case RISCV::BI__builtin_riscv_pm2sub_i16x2:
+ case RISCV::BI__builtin_riscv_pm2sub_i16x4:
+ case RISCV::BI__builtin_riscv_pm2sub_i32x2:
+ ID = Intrinsic::riscv_pm2sub;
+ break;
+ case RISCV::BI__builtin_riscv_pm2sub_x_i16x2:
+ case RISCV::BI__builtin_riscv_pm2sub_x_i16x4:
+ case RISCV::BI__builtin_riscv_pm2sub_x_i32x2:
+ ID = Intrinsic::riscv_pm2sub_x;
+ break;
+ case RISCV::BI__builtin_riscv_pm4addsu_i8x4:
+ case RISCV::BI__builtin_riscv_pm4addsu_i8x8:
+ case RISCV::BI__builtin_riscv_pm4addsu_i16x4:
+ ID = Intrinsic::riscv_pm4addsu;
+ break;
+ case RISCV::BI__builtin_riscv_pm2addsu_i16x2:
+ case RISCV::BI__builtin_riscv_pm2addsu_i16x4:
+ case RISCV::BI__builtin_riscv_pm2addsu_i32x2:
+ ID = Intrinsic::riscv_pm2addsu;
+ break;
+ }
+
+ IntrinsicTypes = {ResultType, Ops[0]->getType()};
+ break;
+ }
+
// Packed Reduction Sum
case RISCV::BI__builtin_riscv_predsum_i8x4_i32:
case RISCV::BI__builtin_riscv_predsum_i16x2_i32:
diff --git a/clang/lib/Headers/riscv_packed_simd.h b/clang/lib/Headers/riscv_packed_simd.h
index 946dd936d49cd..265d2f71be34c 100644
--- a/clang/lib/Headers/riscv_packed_simd.h
+++ b/clang/lib/Headers/riscv_packed_simd.h
@@ -813,6 +813,47 @@ __packed_ternary_builtin_mixed(pmhaccsu_i32x2, int32x2_t, int32x2_t, uint32x2_t,
__packed_ternary_builtin_mixed(pmhraccsu_i16x4, int16x4_t, int16x4_t, uint16x4_t, __builtin_riscv_pmhraccsu_i16x4)
__packed_ternary_builtin_mixed(pmhraccsu_i32x2, int32x2_t, int32x2_t, uint32x2_t, __builtin_riscv_pmhraccsu_i32x2)
+/* Packed Multiplication with Horizontal Addition (32-bit) */
+__packed_binary_builtin_mixed(pm4add_i8x4, int32_t, int8x4_t, int8x4_t, __builtin_riscv_pm4add_i8x4)
+__packed_binary_builtin_mixed(pm2add_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2add_i16x2)
+__packed_binary_builtin_mixed(pm2add_x_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2add_x_i16x2)
+__packed_binary_builtin_mixed(pm4addu_u8x4, uint32_t, uint8x4_t, uint8x4_t, __builtin_riscv_pm4addu_u8x4)
+__packed_binary_builtin_mixed(pm2addu_u16x2, uint32_t, uint16x2_t, uint16x2_t, __builtin_riscv_pm2addu_u16x2)
+__packed_binary_builtin_mixed(pmq2add_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pmq2add_i16x2)
+__packed_binary_builtin_mixed(pmqr2add_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pmqr2add_i16x2)
+__packed_binary_builtin_mixed(pm2sadd_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2sadd_i16x2)
+__packed_binary_builtin_mixed(pm2sadd_x_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2sadd_x_i16x2)
+__packed_binary_builtin_mixed(pm2sub_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2sub_i16x2)
+__packed_binary_builtin_mixed(pm2sub_x_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2sub_x_i16x2)
+__packed_binary_builtin_mixed(pm4addsu_i8x4, int32_t, int8x4_t, uint8x4_t, __builtin_riscv_pm4addsu_i8x4)
+__packed_binary_builtin_mixed(pm2addsu_i16x2, int32_t, int16x2_t, uint16x2_t, __builtin_riscv_pm2addsu_i16x2)
+
+/* Packed Multiplication with Horizontal Addition (64-bit) */
+__packed_binary_builtin_mixed(pm4add_i8x8, int32x2_t, int8x8_t, int8x8_t, __builtin_riscv_pm4add_i8x8)
+__packed_binary_builtin_mixed(pm2add_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2add_i16x4)
+__packed_binary_builtin_mixed(pm2add_x_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2add_x_i16x4)
+__packed_binary_builtin_mixed(pm4addu_u8x8, uint32x2_t, uint8x8_t, uint8x8_t, __builtin_riscv_pm4addu_u8x8)
+__packed_binary_builtin_mixed(pm2addu_u16x4, uint32x2_t, uint16x4_t, uint16x4_t, __builtin_riscv_pm2addu_u16x4)
+__packed_binary_builtin_mixed(pmq2add_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pmq2add_i16x4)
+__packed_binary_builtin_mixed(pmqr2add_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pmqr2add_i16x4)
+__packed_binary_builtin_mixed(pm2sadd_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2sadd_i16x4)
+__packed_binary_builtin_mixed(pm2sadd_x_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2sadd_x_i16x4)
+__packed_binary_builtin_mixed(pm2sub_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2sub_i16x4)
+__packed_binary_builtin_mixed(pm2sub_x_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2sub_x_i16x4)
+__packed_binary_builtin_mixed(pm4addsu_i8x8, int32x2_t, int8x8_t, uint8x8_t, __builtin_riscv_pm4addsu_i8x8)
+__packed_binary_builtin_mixed(pm2addsu_i16x4, int32x2_t, int16x4_t, uint16x4_t, __builtin_riscv_pm2addsu_i16x4)
+__packed_binary_builtin_mixed(pm2add_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pm2add_i32x2)
+__packed_binary_builtin_mixed(pm2add_x_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pm2add_x_i32x2)
+__packed_binary_builtin_mixed(pm2addu_u32x2, uint64_t, uint32x2_t, uint32x2_t, __builtin_riscv_pm2addu_u32x2)
+__packed_binary_builtin_mixed(pmq2add_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pmq2add_i32x2)
+__packed_binary_builtin_mixed(pm2sub_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pm2sub_i32x2)
+__packed_binary_builtin_mixed(pm2sub_x_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pm2sub_x_i32x2)
+__packed_binary_builtin_mixed(pm2addsu_i32x2, int64_t, int32x2_t, uint32x2_t, __builtin_riscv_pm2addsu_i32x2)
+__packed_binary_builtin_mixed(pmqr2add_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pmqr2add_i32x2)
+__packed_binary_builtin_mixed(pm4add_i16x4, int64_t, int16x4_t, int16x4_t, __builtin_riscv_pm4add_i16x4)
+__packed_binary_builtin_mixed(pm4addu_u16x4, uint64_t, uint16x4_t, uint16x4_t, __builtin_riscv_pm4addu_u16x4)
+__packed_binary_builtin_mixed(pm4addsu_i16x4, int64_t, int16x4_t, uint16x4_t, __builtin_riscv_pm4addsu_i16x4)
+
/* Packed Absolute Difference Sum (32-bit) */
__packed_abdsum(pabdsumu_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumu_u8x4_u32)
__packed_ternary_builtin_cast(pabdsumau_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumau_u8x4_u32)
diff --git a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
index 28e3a8911739d..32e4fa767f69e 100644
--- a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
+++ b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
@@ -3532,3 +3532,288 @@ int64_t test_mqracc_w01_i64(int64_t rd, int32x2_t a, int32x2_t b) {
int64_t test_mqracc_w11_i64(int64_t rd, int32x2_t a, int32x2_t b) {
return __riscv_mqracc_w11_i64(rd, a, b);
}
+
+// CHECK-LABEL: test_pm4add_i8x4:
+// RV32: pm4add.b
+// RV64: pm4add.b
+int32_t test_pm4add_i8x4(int8x4_t rs1, int8x4_t rs2) {
+ return __riscv_pm4add_i8x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2add_i16x2:
+// RV32: pm2add.h
+// RV64: pm2add.h
+int32_t test_pm2add_i16x2(int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2add_i16x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2add_x_i16x2:
+// RV32: pm2add.hx
+// RV64: pm2add.hx
+int32_t test_pm2add_x_i16x2(int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2add_x_i16x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm4addu_u8x4:
+// RV32: pm4addu.b
+// RV64: pm4addu.b
+uint32_t test_pm4addu_u8x4(uint8x4_t rs1, uint8x4_t rs2) {
+ return __riscv_pm4addu_u8x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2addu_u16x2:
+// RV32: pm2addu.h
+// RV64: pm2addu.h
+uint32_t test_pm2addu_u16x2(uint16x2_t rs1, uint16x2_t rs2) {
+ return __riscv_pm2addu_u16x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmq2add_i16x2:
+// RV32: pmq2add.h
+// RV64: pmq2add.h
+int32_t test_pmq2add_i16x2(int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pmq2add_i16x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmqr2add_i16x2:
+// RV32: pmqr2add.h
+// RV64: pmqr2add.h
+int32_t test_pmqr2add_i16x2(int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pmqr2add_i16x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2sadd_i16x2:
+// RV32: pm2sadd.h
+// RV64: pm2sadd.h
+int32_t test_pm2sadd_i16x2(int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2sadd_i16x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2sadd_x_i16x2:
+// RV32: pm2sadd.hx
+// RV64: pm2sadd.hx
+int32_t test_pm2sadd_x_i16x2(int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2sadd_x_i16x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2sub_i16x2:
+// RV32: pm2sub.h
+// RV64: pm2sub.h
+int32_t test_pm2sub_i16x2(int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2sub_i16x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2sub_x_i16x2:
+// RV32: pm2sub.hx
+// RV64: pm2sub.hx
+int32_t test_pm2sub_x_i16x2(int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2sub_x_i16x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm4addsu_i8x4:
+// RV32: pm4addsu.b
+// RV64: pm4addsu.b
+int32_t test_pm4addsu_i8x4(int8x4_t rs1, uint8x4_t rs2) {
+ return __riscv_pm4addsu_i8x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2addsu_i16x2:
+// RV32: pm2addsu.h
+// RV64: pm2addsu.h
+int32_t test_pm2addsu_i16x2(int16x2_t rs1, uint16x2_t rs2) {
+ return __riscv_pm2addsu_i16x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm4add_i8x8:
+// RV32: pm4add.b
+// RV32: pm4add.b
+// RV64: pm4add.b
+int32x2_t test_pm4add_i8x8(int8x8_t rs1, int8x8_t rs2) {
+ return __riscv_pm4add_i8x8(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2add_i16x4:
+// RV32: pm2add.h
+// RV32: pm2add.h
+// RV64: pm2add.h
+int32x2_t test_pm2add_i16x4(int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pm2add_i16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2add_x_i16x4:
+// RV32: pm2add.hx
+// RV32: pm2add.hx
+// RV64: pm2add.hx
+int32x2_t test_pm2add_x_i16x4(int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pm2add_x_i16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm4addu_u8x8:
+// RV32: pm4addu.b
+// RV32: pm4addu.b
+// RV64: pm4addu.b
+uint32x2_t test_pm4addu_u8x8(uint8x8_t rs1, uint8x8_t rs2) {
+ return __riscv_pm4addu_u8x8(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2addu_u16x4:
+// RV32: pm2addu.h
+// RV32: pm2addu.h
+// RV64: pm2addu.h
+uint32x2_t test_pm2addu_u16x4(uint16x4_t rs1, uint16x4_t rs2) {
+ return __riscv_pm2addu_u16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmq2add_i16x4:
+// RV32: pmq2add.h
+// RV32: pmq2add.h
+// RV64: pmq2add.h
+int32x2_t test_pmq2add_i16x4(int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pmq2add_i16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmqr2add_i16x4:
+// RV32: pmqr2add.h
+// RV32: pmqr2add.h
+// RV64: pmqr2add.h
+int32x2_t test_pmqr2add_i16x4(int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pmqr2add_i16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2sadd_i16x4:
+// RV32: pm2sadd.h
+// RV32: pm2sadd.h
+// RV64: pm2sadd.h
+int32x2_t test_pm2sadd_i16x4(int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pm2sadd_i16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2sadd_x_i16x4:
+// RV32: pm2sadd.hx
+// RV32: pm2sadd.hx
+// RV64: pm2sadd.hx
+int32x2_t test_pm2sadd_x_i16x4(int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pm2sadd_x_i16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2sub_i16x4:
+// RV32: pm2sub.h
+// RV32: pm2sub.h
+// RV64: pm2sub.h
+int32x2_t test_pm2sub_i16x4(int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pm2sub_i16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2sub_x_i16x4:
+// RV32: pm2sub.hx
+// RV32: pm2sub.hx
+// RV64: pm2sub.hx
+int32x2_t test_pm2sub_x_i16x4(int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pm2sub_x_i16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm4addsu_i8x8:
+// RV32: pm4addsu.b
+// RV32: pm4addsu.b
+// RV64: pm4addsu.b
+int32x2_t test_pm4addsu_i8x8(int8x8_t rs1, uint8x8_t rs2) {
+ return __riscv_pm4addsu_i8x8(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2addsu_i16x4:
+// RV32: pm2addsu.h
+// RV32: pm2addsu.h
+// RV64: pm2addsu.h
+int32x2_t test_pm2addsu_i16x4(int16x4_t rs1, uint16x4_t rs2) {
+ return __riscv_pm2addsu_i16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2add_i32x2:
+// RV32: wmul
+// RV32: wmacc
+// RV64: pm2add.w
+int64_t test_pm2add_i32x2(int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_pm2add_i32x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2add_x_i32x2:
+// RV32: wmul
+// RV32: wmacc
+// RV64: pm2add.wx
+int64_t test_pm2add_x_i32x2(int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_pm2add_x_i32x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2addu_u32x2:
+// RV32: wmulu
+// RV32: wmaccu
+// RV64: pm2addu.w
+uint64_t test_pm2addu_u32x2(uint32x2_t rs1, uint32x2_t rs2) {
+ return __riscv_pm2addu_u32x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmq2add_i32x2:
+// RV32: mqwacc
+// RV32: mqwacc
+// RV64: pmq2add.w
+int64_t test_pmq2add_i32x2(int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_pmq2add_i32x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmqr2add_i32x2:
+// RV32: mqrwacc
+// RV32: mqrwacc
+// RV64: pmqr2add.w
+int64_t test_pmqr2add_i32x2(int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_pmqr2add_i32x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2sub_i32x2:
+// RV32: wmul
+// RV32: wmul
+// RV32: subd
+// RV64: pm2sub.w
+int64_t test_pm2sub_i32x2(int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_pm2sub_i32x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2sub_x_i32x2:
+// RV32: wmul
+// RV32: wmul
+// RV32: subd
+// RV64: pm2sub.wx
+int64_t test_pm2sub_x_i32x2(int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_pm2sub_x_i32x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2addsu_i32x2:
+// RV32: wmulsu
+// RV32: wmaccsu
+// RV64: pm2addsu.w
+int64_t test_pm2addsu_i32x2(int32x2_t rs1, uint32x2_t rs2) {
+ return __riscv_pm2addsu_i32x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm4add_i16x4:
+// RV32: pm2wadd.h
+// RV32: pm2wadda.h
+// RV64: pm4add.h
+int64_t test_pm4add_i16x4(int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pm4add_i16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm4addu_u16x4:
+// RV32: pm2waddu.h
+// RV32: pm2waddau.h
+// RV64: pm4addu.h
+uint64_t test_pm4addu_u16x4(uint16x4_t rs1, uint16x4_t rs2) {
+ return __riscv_pm4addu_u16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm4addsu_i16x4:
+// RV32: pm2waddsu.h
+// RV32: pm2waddasu.h
+// RV64: pm4addsu.h
+int64_t test_pm4addsu_i16x4(int16x4_t rs1, uint16x4_t rs2) {
+ return __riscv_pm4addsu_i16x4(rs1, rs2);
+}
diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td
index 9c810ecc2a624..6320b7710cd18 100644
--- a/llvm/include/llvm/IR/IntrinsicsRISCV.td
+++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td
@@ -2125,6 +2125,25 @@ class RVPBinaryIntrinsic
def int_riscv_pmhaccsu : RVPTernaryIntrinsic;
def int_riscv_pmhraccsu : RVPTernaryIntrinsic;
+ // Packed Multiplication with Horizontal Addition.
+ class RVPHorizontalMulIntrinsic
+ : DefaultAttrsIntrinsic<[llvm_any_ty],
+ [llvm_anyvector_ty, LLVMMatchType<1>],
+ [IntrNoMem, IntrSpeculatable]>;
+ def int_riscv_pm4add : RVPHorizontalMulIntrinsic;
+ def int_riscv_pm2add : RVPHorizontalMulIntrinsic;
+ def int_riscv_pm2add_x : RVPHorizontalMulIntrinsic;
+ def int_riscv_pm4addu : RVPHorizontalMulIntrinsic;
+ def int_riscv_pm2addu : RVPHorizontalMulIntrinsic;
+ def int_riscv_pmq2add : RVPHorizontalMulIntrinsic;
+ def int_riscv_pmqr2add : RVPHorizontalMulIntrinsic;
+ def int_riscv_pm2sadd : RVPHorizontalMulIntrinsic;
+ def int_riscv_pm2sadd_x : RVPHorizontalMulIntrinsic;
+ def int_riscv_pm2sub : RVPHorizontalMulIntrinsic;
+ def int_riscv_pm2sub_x : RVPHorizontalMulIntrinsic;
+ def int_riscv_pm4addsu : RVPHorizontalMulIntrinsic;
+ def int_riscv_pm2addsu : RVPHorizontalMulIntrinsic;
+
// Packed Absolute Difference Sum.
def int_riscv_pabdsumu
: DefaultAttrsIntrinsic<[llvm_anyint_ty],
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index fb403aa3bc274..8196fbff2ec1a 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -12329,6 +12329,40 @@ static unsigned getRVPQFormatAccOpcode(Intrinsic::ID IntNo) {
}
}
+static unsigned getRVPHorizontalMulOpcode(unsigned IntNo) {
+ switch (IntNo) {
+ default:
+ llvm_unreachable(
+ "Unexpected RISC-V packed horizontal multiply intrinsic");
+ case Intrinsic::riscv_pm4add:
+ return RISCVISD::PM4ADD;
+ case Intrinsic::riscv_pm2add:
+ return RISCVISD::PM2ADD;
+ case Intrinsic::riscv_pm2add_x:
+ return RISCVISD::PM2ADD_X;
+ case Intrinsic::riscv_pm4addu:
+ return RISCVISD::PM4ADDU;
+ case Intrinsic::riscv_pm2addu:
+ return RISCVISD::PM2ADDU;
+ case Intrinsic::riscv_pmq2add:
+ return RISCVISD::PMQ2ADD;
+ case Intrinsic::riscv_pmqr2add:
+ return RISCVISD::PMQR2ADD;
+ case Intrinsic::riscv_pm2sadd:
+ return RISCVISD::PM2SADD;
+ case Intrinsic::riscv_pm2sadd_x:
+ return RISCVISD::PM2SADD_X;
+ case Intrinsic::riscv_pm2sub:
+ return RISCVISD::PM2SUB;
+ case Intrinsic::riscv_pm2sub_x:
+ return RISCVISD::PM2SUB_X;
+ case Intrinsic::riscv_pm4addsu:
+ return RISCVISD::PM4ADDSU;
+ case Intrinsic::riscv_pm2addsu:
+ return RISCVISD::PM2ADDSU;
+ }
+}
+
SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
SelectionDAG &DAG) const {
unsigned IntNo = Op.getConstantOperandVal(0);
@@ -12779,6 +12813,36 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
return DAG.getNode(MulOpc, DL, VT, Rd, Rs1, Rs2);
}
+ case Intrinsic::riscv_pm4add:
+ case Intrinsic::riscv_pm2add:
+ case Intrinsic::riscv_pm2add_x:
+ case Intrinsic::riscv_pm4addu:
+ case Intrinsic::riscv_pm2addu:
+ case Intrinsic::riscv_pmq2add:
+ case Intrinsic::riscv_pmqr2add:
+ case Intrinsic::riscv_pm2sadd:
+ case Intrinsic::riscv_pm2sadd_x:
+ case Intrinsic::riscv_pm2sub:
+ case Intrinsic::riscv_pm2sub_x:
+ case Intrinsic::riscv_pm4addsu:
+ case Intrinsic::riscv_pm2addsu: {
+ EVT VT = Op.getValueType();
+ unsigned Opc = getRVPHorizontalMulOpcode(IntNo);
+ SDValue Rs1 = Op.getOperand(1);
+ SDValue Rs2 = Op.getOperand(2);
+
+ // RV32 applies the 32-bit instruction independently to both halves of a
+ // 64-bit packed input.
+ if (!Subtarget.is64Bit() && VT == MVT::v2i32) {
+ auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
+ auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
+ SDValue Lo = DAG.getNode(Opc, DL, MVT::i32, Rs1Lo, Rs2Lo);
+ SDValue Hi = DAG.getNode(Opc, DL, MVT::i32, Rs1Hi, Rs2Hi);
+ return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Lo, Hi);
+ }
+
+ return DAG.getNode(Opc, DL, VT, Rs1, Rs2);
+ }
case Intrinsic::riscv_pmerge: {
EVT VT = Op.getValueType();
auto buildMerge = [&](SDValue Rs1, SDValue Rs2, SDValue Mask,
@@ -16870,6 +16934,42 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
Results.push_back(DAG.getExtractSubvector(DL, VT, Res, 0));
return;
}
+ case Intrinsic::riscv_pm4add:
+ case Intrinsic::riscv_pm2add:
+ case Intrinsic::riscv_pm2add_x:
+ case Intrinsic::riscv_pm4addu:
+ case Intrinsic::riscv_pm2addu:
+ case Intrinsic::riscv_pmq2add:
+ case Intrinsic::riscv_pmqr2add:
+ case Intrinsic::riscv_pm2sadd:
+ case Intrinsic::riscv_pm2sadd_x:
+ case Intrinsic::riscv_pm2sub:
+ case Intrinsic::riscv_pm2sub_x:
+ case Intrinsic::riscv_pm4addsu:
+ case Intrinsic::riscv_pm2addsu: {
+ MVT VT = N->getSimpleValueType(0);
+ unsigned Opc = getRVPHorizontalMulOpcode(IntNo);
+ SDValue Rs1 = N->getOperand(1);
+ SDValue Rs2 = N->getOperand(2);
+
+ if (!Subtarget.is64Bit() && VT == MVT::i64) {
+ SDValue Pair = DAG.getNode(Opc, DL, MVT::v2i32, Rs1, Rs2);
+ Results.push_back(DAG.getBitcast(MVT::i64, Pair));
+ return;
+ }
+
+ assert(Subtarget.is64Bit() && VT == MVT::i32 &&
+ "Unexpected horizontal multiply legalization");
+ MVT SrcVT = Rs1.getSimpleValueType();
+ MVT WideSrcVT = SrcVT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16;
+ Rs1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideSrcVT, Rs1,
+ DAG.getUNDEF(SrcVT));
+ Rs2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideSrcVT, Rs2,
+ DAG.getUNDEF(SrcVT));
+ SDValue Wide = DAG.getNode(Opc, DL, MVT::v2i32, Rs1, Rs2);
+ Results.push_back(DAG.getExtractVectorElt(DL, MVT::i32, Wide, 0));
+ return;
+ }
case Intrinsic::riscv_paadd:
case Intrinsic::riscv_paaddu:
case Intrinsic::riscv_pasub:
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
index 06cf949cca56b..807792928a336 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
@@ -1935,6 +1935,23 @@ def riscv_mhraccu : RVSDNode<"MHRACCU", SDT_RISCVSameTernary>;
def riscv_mhaccsu : RVSDNode<"MHACCSU", SDT_RISCVSameTernary>;
def riscv_mhraccsu : RVSDNode<"MHRACCSU", SDT_RISCVSameTernary>;
+// Multiply packed elements and horizontally combine each adjacent group.
+def SDT_RISCVHorizontalMul
+ : SDTypeProfile<1, 2, [SDTCisVec<1>, SDTCisSameAs<1, 2>]>;
+def riscv_pm4add : RVSDNode<"PM4ADD", SDT_RISCVHorizontalMul>;
+def riscv_pm2add : RVSDNode<"PM2ADD", SDT_RISCVHorizontalMul>;
+def riscv_pm2add_x : RVSDNode<"PM2ADD_X", SDT_RISCVHorizontalMul>;
+def riscv_pm4addu : RVSDNode<"PM4ADDU", SDT_RISCVHorizontalMul>;
+def riscv_pm2addu : RVSDNode<"PM2ADDU", SDT_RISCVHorizontalMul>;
+def riscv_pmq2add : RVSDNode<"PMQ2ADD", SDT_RISCVHorizontalMul>;
+def riscv_pmqr2add : RVSDNode<"PMQR2ADD", SDT_RISCVHorizontalMul>;
+def riscv_pm2sadd : RVSDNode<"PM2SADD", SDT_RISCVHorizontalMul>;
+def riscv_pm2sadd_x : RVSDNode<"PM2SADD_X", SDT_RISCVHorizontalMul>;
+def riscv_pm2sub : RVSDNode<"PM2SUB", SDT_RISCVHorizontalMul>;
+def riscv_pm2sub_x : RVSDNode<"PM2SUB_X", SDT_RISCVHorizontalMul>;
+def riscv_pm4addsu : RVSDNode<"PM4ADDSU", SDT_RISCVHorizontalMul>;
+def riscv_pm2addsu : RVSDNode<"PM2ADDSU", SDT_RISCVHorizontalMul>;
+
// "Q-format" multiplication
def riscv_mulq : RVSDNode<"MULQ", SDTIntBinOp, [SDNPCommutative]>;
def riscv_mulqr : RVSDNode<"MULQR", SDTIntBinOp, [SDNPCommutative]>;
@@ -2063,6 +2080,11 @@ def RISCVBuildPairGPRVec : RVSDNode<"BuildPairGPRVec",
SDT_RISCVBuildPairGPRVec>;
def RISCVSplitGPRVec : RVSDNode<"SplitGPRVec", SDT_RISCVSplitGPRVec>;
+class PatHorizontalMul<SDPatternOperator OpNode, RVInst Inst,
+ ValueType ResultVT, ValueType SrcVT>
+ : Pat<(ResultVT (OpNode (SrcVT GPR:$rs1), (SrcVT GPR:$rs2))),
+ (Inst GPR:$rs1, GPR:$rs2)>;
+
// (rs1 << 1) + rs2
class PatPSh1Add<RVInst Inst, ValueType vt>
: Pat<(vt (add (riscv_pshl (vt GPR:$rs1), (XLenVT 1)), (vt GPR:$rs2))),
@@ -2340,6 +2362,97 @@ let append Predicates = [IsRV32] in {
def : PatGprGpr<riscv_mulq, MULQ, i32>;
def : PatGprGpr<riscv_mulqr, MULQR, i32>;
+ // Packed multiplication with horizontal addition patterns.
+ def : PatHorizontalMul<riscv_pm4add, PM4ADD_B, i32, v4i8>;
+ def : PatHorizontalMul<riscv_pm2add, PM2ADD_H, i32, v2i16>;
+ def : PatHorizontalMul<riscv_pm2add_x, PM2ADD_HX, i32, v2i16>;
+ def : PatHorizontalMul<riscv_pm4addu, PM4ADDU_B, i32, v4i8>;
+ def : PatHorizontalMul<riscv_pm2addu, PM2ADDU_H, i32, v2i16>;
+ def : PatHorizontalMul<riscv_pmq2add, PMQ2ADD_H, i32, v2i16>;
+ def : PatHorizontalMul<riscv_pmqr2add, PMQR2ADD_H, i32, v2i16>;
+ def : PatHorizontalMul<riscv_pm2sadd, PM2SADD_H, i32, v2i16>;
+ def : PatHorizontalMul<riscv_pm2sadd_x, PM2SADD_HX, i32, v2i16>;
+ def : PatHorizontalMul<riscv_pm2sub, PM2SUB_H, i32, v2i16>;
+ def : PatHorizontalMul<riscv_pm2sub_x, PM2SUB_HX, i32, v2i16>;
+ def : PatHorizontalMul<riscv_pm4addsu, PM4ADDSU_B, i32, v4i8>;
+ def : PatHorizontalMul<riscv_pm2addsu, PM2ADDSU_H, i32, v2i16>;
+
+ // Select the RV32 v2i32 result produced by type legalization into the
+ // instruction sequences specified for the 64-bit intrinsics.
+ def : Pat<(v2i32 (riscv_pm2add (v2i32 GPRPair:$rs1),
+ (v2i32 GPRPair:$rs2))),
+ (WMACC (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)),
+ (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>;
+ def : Pat<(v2i32 (riscv_pm2add_x (v2i32 GPRPair:$rs1),
+ (v2i32 GPRPair:$rs2))),
+ (WMACC (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd)),
+ (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even))>;
+ def : Pat<(v2i32 (riscv_pm2addu (v2i32 GPRPair:$rs1),
+ (v2i32 GPRPair:$rs2))),
+ (WMACCU (WMULU (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)),
+ (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>;
+ def : Pat<(v2i32 (riscv_pmq2add (v2i32 GPRPair:$rs1),
+ (v2i32 GPRPair:$rs2))),
+ (MQWACC (v2i32 (MQWACC
+ (v2i32 (BuildGPRPair (i32 X0), (i32 X0))),
+ (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even))),
+ (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>;
+ def : Pat<(v2i32 (riscv_pmqr2add (v2i32 GPRPair:$rs1),
+ (v2i32 GPRPair:$rs2))),
+ (MQRWACC (v2i32 (MQRWACC
+ (v2i32 (BuildGPRPair (i32 X0), (i32 X0))),
+ (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even))),
+ (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>;
+ def : Pat<(v2i32 (riscv_pm2sub (v2i32 GPRPair:$rs1),
+ (v2i32 GPRPair:$rs2))),
+ (SUBD (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)),
+ (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd)))>;
+ def : Pat<(v2i32 (riscv_pm2sub_x (v2i32 GPRPair:$rs1),
+ (v2i32 GPRPair:$rs2))),
+ (SUBD (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd)),
+ (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)))>;
+ def : Pat<(v2i32 (riscv_pm2addsu (v2i32 GPRPair:$rs1),
+ (v2i32 GPRPair:$rs2))),
+ (WMACCSU (WMULSU (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)),
+ (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>;
+ def : Pat<(v2i32 (riscv_pm4add (v4i16 GPRPair:$rs1),
+ (v4i16 GPRPair:$rs2))),
+ (PM2WADDA_H
+ (PM2WADD_H (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)),
+ (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>;
+ def : Pat<(v2i32 (riscv_pm4addu (v4i16 GPRPair:$rs1),
+ (v4i16 GPRPair:$rs2))),
+ (PM2WADDAU_H
+ (PM2WADDU_H (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)),
+ (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>;
+ def : Pat<(v2i32 (riscv_pm4addsu (v4i16 GPRPair:$rs1),
+ (v4i16 GPRPair:$rs2))),
+ (PM2WADDASU_H
+ (PM2WADDSU_H (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)),
+ (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
+ (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>;
+
// Halfword multiply patterns where one operand is a sext.h or zext.h and
// the other is a sext.h or zext.h or is known to be sign/zero-extended. We
// prefer plain mul when both operands are known to be sign/zero-extended.
@@ -2948,6 +3061,33 @@ let append Predicates = [IsRV64] in {
def : PatGprGpr<riscv_mulq, PMULQ_W, v2i32>;
def : PatGprGpr<riscv_mulqr, PMULQR_W, v2i32>;
+ // Packed multiplication with horizontal addition patterns.
+ def : PatHorizontalMul<riscv_pm4add, PM4ADD_B, v2i32, v8i8>;
+ def : PatHorizontalMul<riscv_pm2add, PM2ADD_H, v2i32, v4i16>;
+ def : PatHorizontalMul<riscv_pm2add_x, PM2ADD_HX, v2i32, v4i16>;
+ def : PatHorizontalMul<riscv_pm4addu, PM4ADDU_B, v2i32, v8i8>;
+ def : PatHorizontalMul<riscv_pm2addu, PM2ADDU_H, v2i32, v4i16>;
+ def : PatHorizontalMul<riscv_pmq2add, PMQ2ADD_H, v2i32, v4i16>;
+ def : PatHorizontalMul<riscv_pmqr2add, PMQR2ADD_H, v2i32, v4i16>;
+ def : PatHorizontalMul<riscv_pm2sadd, PM2SADD_H, v2i32, v4i16>;
+ def : PatHorizontalMul<riscv_pm2sadd_x, PM2SADD_HX, v2i32, v4i16>;
+ def : PatHorizontalMul<riscv_pm2sub, PM2SUB_H, v2i32, v4i16>;
+ def : PatHorizontalMul<riscv_pm2sub_x, PM2SUB_HX, v2i32, v4i16>;
+ def : PatHorizontalMul<riscv_pm4addsu, PM4ADDSU_B, v2i32, v8i8>;
+ def : PatHorizontalMul<riscv_pm2addsu, PM2ADDSU_H, v2i32, v4i16>;
+
+ def : PatHorizontalMul<riscv_pm2add, PM2ADD_W, i64, v2i32>;
+ def : PatHorizontalMul<riscv_pm2add_x, PM2ADD_WX, i64, v2i32>;
+ def : PatHorizontalMul<riscv_pm2addu, PM2ADDU_W, i64, v2i32>;
+ def : PatHorizontalMul<riscv_pmq2add, PMQ2ADD_W, i64, v2i32>;
+ def : PatHorizontalMul<riscv_pmqr2add, PMQR2ADD_W, i64, v2i32>;
+ def : PatHorizontalMul<riscv_pm2sub, PM2SUB_W, i64, v2i32>;
+ def : PatHorizontalMul<riscv_pm2sub_x, PM2SUB_WX, i64, v2i32>;
+ def : PatHorizontalMul<riscv_pm2addsu, PM2ADDSU_W, i64, v2i32>;
+ def : PatHorizontalMul<riscv_pm4add, PM4ADD_H, i64, v4i16>;
+ def : PatHorizontalMul<riscv_pm4addu, PM4ADDU_H, i64, v4i16>;
+ def : PatHorizontalMul<riscv_pm4addsu, PM4ADDSU_H, i64, v4i16>;
+
// 8/16/32-bit multiply low patterns
// FIXME custom lower
def : Pat<(v8i8 (mul GPR:$rs1, GPR:$rs2)),
diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-32.ll b/llvm/test/CodeGen/RISCV/rvp-simd-32.ll
index f1209617b6a90..65294ff5b9b3b 100644
--- a/llvm/test/CodeGen/RISCV/rvp-simd-32.ll
+++ b/llvm/test/CodeGen/RISCV/rvp-simd-32.ll
@@ -3240,3 +3240,120 @@ define i32 @test_mqracc_h11_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) {
%r = call i32 @llvm.riscv.mqracc.h11.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b)
ret i32 %r
}
+
+define i32 @test_pm4add_v4i8_i32(<4 x i8> %a, <4 x i8> %b) {
+; CHECK-LABEL: test_pm4add_v4i8_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm4add.b a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm4add.i32.v4i8(<4 x i8> %a, <4 x i8> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2add_v2i16_i32(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2add_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2add.h a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2add.i32.v2i16(<2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2add_x_v2i16_i32(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2add_x_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2add.hx a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2add.x.i32.v2i16(<2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm4addu_v4i8_i32(<4 x i8> %a, <4 x i8> %b) {
+; CHECK-LABEL: test_pm4addu_v4i8_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm4addu.b a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm4addu.i32.v4i8(<4 x i8> %a, <4 x i8> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2addu_v2i16_i32(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2addu_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2addu.h a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2addu.i32.v2i16(<2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pmq2add_v2i16_i32(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pmq2add_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pmq2add.h a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pmq2add.i32.v2i16(<2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pmqr2add_v2i16_i32(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pmqr2add_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pmqr2add.h a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pmqr2add.i32.v2i16(<2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2sadd_v2i16_i32(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2sadd_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2sadd.h a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2sadd.i32.v2i16(<2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2sadd_x_v2i16_i32(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2sadd_x_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2sadd.hx a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2sadd.x.i32.v2i16(<2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2sub_v2i16_i32(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2sub_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2sub.h a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2sub.i32.v2i16(<2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2sub_x_v2i16_i32(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2sub_x_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2sub.hx a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2sub.x.i32.v2i16(<2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm4addsu_v4i8_i32(<4 x i8> %a, <4 x i8> %b) {
+; CHECK-LABEL: test_pm4addsu_v4i8_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm4addsu.b a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm4addsu.i32.v4i8(<4 x i8> %a, <4 x i8> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2addsu_v2i16_i32(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2addsu_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2addsu.h a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2addsu.i32.v2i16(<2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
index 51b86057ccdbe..ad2d4e2dbeb11 100644
--- a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
+++ b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
@@ -7113,3 +7113,378 @@ define i64 @test_mqracc_w11_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) {
%r = call i64 @llvm.riscv.mqracc.w11.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b)
ret i64 %r
}
+
+define <2 x i32> @test_pm4add_v8i8_v2i32(<8 x i8> %a, <8 x i8> %b) {
+; RV32-LABEL: test_pm4add_v8i8_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm4add.b a1, a1, a3
+; RV32-NEXT: pm4add.b a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm4add_v8i8_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm4add.b a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm4add.v2i32.v8i8(<8 x i8> %a, <8 x i8> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm2add_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2add_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2add.h a1, a1, a3
+; RV32-NEXT: pm2add.h a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2add_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2add.h a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2add.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm2add_x_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2add_x_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2add.hx a1, a1, a3
+; RV32-NEXT: pm2add.hx a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2add_x_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2add.hx a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2add.x.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm4addu_v8i8_v2i32(<8 x i8> %a, <8 x i8> %b) {
+; RV32-LABEL: test_pm4addu_v8i8_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm4addu.b a1, a1, a3
+; RV32-NEXT: pm4addu.b a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm4addu_v8i8_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm4addu.b a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm4addu.v2i32.v8i8(<8 x i8> %a, <8 x i8> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm2addu_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2addu_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2addu.h a1, a1, a3
+; RV32-NEXT: pm2addu.h a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2addu_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2addu.h a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2addu.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pmq2add_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pmq2add_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pmq2add.h a1, a1, a3
+; RV32-NEXT: pmq2add.h a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pmq2add_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pmq2add.h a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pmq2add.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pmqr2add_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pmqr2add_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pmqr2add.h a1, a1, a3
+; RV32-NEXT: pmqr2add.h a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pmqr2add_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pmqr2add.h a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pmqr2add.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm2sadd_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2sadd_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2sadd.h a1, a1, a3
+; RV32-NEXT: pm2sadd.h a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2sadd_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2sadd.h a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2sadd.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm2sadd_x_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2sadd_x_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2sadd.hx a1, a1, a3
+; RV32-NEXT: pm2sadd.hx a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2sadd_x_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2sadd.hx a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2sadd.x.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm2sub_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2sub_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2sub.h a1, a1, a3
+; RV32-NEXT: pm2sub.h a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2sub_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2sub.h a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2sub.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm2sub_x_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2sub_x_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2sub.hx a1, a1, a3
+; RV32-NEXT: pm2sub.hx a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2sub_x_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2sub.hx a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2sub.x.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm4addsu_v8i8_v2i32(<8 x i8> %a, <8 x i8> %b) {
+; RV32-LABEL: test_pm4addsu_v8i8_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm4addsu.b a1, a1, a3
+; RV32-NEXT: pm4addsu.b a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm4addsu_v8i8_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm4addsu.b a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm4addsu.v2i32.v8i8(<8 x i8> %a, <8 x i8> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm2addsu_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2addsu_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2addsu.h a1, a1, a3
+; RV32-NEXT: pm2addsu.h a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2addsu_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2addsu.h a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2addsu.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define i64 @test_pm2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pm2add_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mv a4, a1
+; RV32-NEXT: wmul a0, a0, a2
+; RV32-NEXT: wmacc a0, a4, a3
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2add_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2add.w a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2add.i64.v2i32(<2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm2add_x_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pm2add_x_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mv a4, a1
+; RV32-NEXT: wmul a0, a0, a3
+; RV32-NEXT: wmacc a0, a4, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2add_x_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2add.wx a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2add.x.i64.v2i32(<2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm2addu_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pm2addu_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mv a4, a1
+; RV32-NEXT: wmulu a0, a0, a2
+; RV32-NEXT: wmaccu a0, a4, a3
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2addu_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2addu.w a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2addu.i64.v2i32(<2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pmq2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pmq2add_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: li a4, 0
+; RV32-NEXT: li a5, 0
+; RV32-NEXT: mqwacc a4, a0, a2
+; RV32-NEXT: mqwacc a4, a1, a3
+; RV32-NEXT: mvd a0, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pmq2add_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pmq2add.w a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pmq2add.i64.v2i32(<2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pmqr2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pmqr2add_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: li a4, 0
+; RV32-NEXT: li a5, 0
+; RV32-NEXT: mqrwacc a4, a0, a2
+; RV32-NEXT: mqrwacc a4, a1, a3
+; RV32-NEXT: mvd a0, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pmqr2add_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pmqr2add.w a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pmqr2add.i64.v2i32(<2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm2sub_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pm2sub_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: wmul a4, a1, a3
+; RV32-NEXT: wmul a0, a0, a2
+; RV32-NEXT: subd a0, a0, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2sub_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2sub.w a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2sub.i64.v2i32(<2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm2sub_x_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pm2sub_x_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: wmul a4, a1, a2
+; RV32-NEXT: wmul a0, a0, a3
+; RV32-NEXT: subd a0, a0, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2sub_x_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2sub.wx a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2sub.x.i64.v2i32(<2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm2addsu_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pm2addsu_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mv a4, a1
+; RV32-NEXT: wmulsu a0, a0, a2
+; RV32-NEXT: wmaccsu a0, a4, a3
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2addsu_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2addsu.w a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2addsu.i64.v2i32(<2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm4add_v4i16_i64(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm4add_v4i16_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mv a4, a1
+; RV32-NEXT: pm2wadd.h a0, a0, a2
+; RV32-NEXT: pm2wadda.h a0, a4, a3
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm4add_v4i16_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm4add.h a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm4add.i64.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm4addu_v4i16_i64(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm4addu_v4i16_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mv a4, a1
+; RV32-NEXT: pm2waddu.h a0, a0, a2
+; RV32-NEXT: pm2waddau.h a0, a4, a3
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm4addu_v4i16_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm4addu.h a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm4addu.i64.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm4addsu_v4i16_i64(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm4addsu_v4i16_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mv a4, a1
+; RV32-NEXT: pm2waddsu.h a0, a0, a2
+; RV32-NEXT: pm2waddasu.h a0, a4, a3
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm4addsu_v4i16_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm4addsu.h a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm4addsu.i64.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret i64 %r
+}
>From 76e3e5a5294cf87738c562459fa3cb24fb1847c8 Mon Sep 17 00:00:00 2001
From: XChy <xxs_chy at outlook.com>
Date: Mon, 24 Aug 2026 22:56:04 +0800
Subject: [PATCH 2/4] format
---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 8196fbff2ec1a..e2969e965119c 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -12332,8 +12332,7 @@ static unsigned getRVPQFormatAccOpcode(Intrinsic::ID IntNo) {
static unsigned getRVPHorizontalMulOpcode(unsigned IntNo) {
switch (IntNo) {
default:
- llvm_unreachable(
- "Unexpected RISC-V packed horizontal multiply intrinsic");
+ llvm_unreachable("Unexpected RISC-V packed horizontal multiply intrinsic");
case Intrinsic::riscv_pm4add:
return RISCVISD::PM4ADD;
case Intrinsic::riscv_pm2add:
>From 9482d506095661b1f38c65987cc082c0e0a9c535 Mon Sep 17 00:00:00 2001
From: XChy <xxs_chy at outlook.com>
Date: Tue, 25 Aug 2026 01:22:01 +0800
Subject: [PATCH 3/4] resolve comments
---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 68 +++++++++-
llvm/lib/Target/RISCV/RISCVInstrInfoP.td | 131 ++++++++------------
llvm/test/CodeGen/RISCV/rvp-simd-64.ll | 30 +++--
3 files changed, 135 insertions(+), 94 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index e2969e965119c..b64998d98a913 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -12362,6 +12362,70 @@ static unsigned getRVPHorizontalMulOpcode(unsigned IntNo) {
}
}
+static SDValue lowerRV32HorizontalMul64(unsigned IntNo, SDValue Rs1,
+ SDValue Rs2, const SDLoc &DL,
+ SelectionDAG &DAG) {
+ if (Rs1.getSimpleValueType() == MVT::v4i16) {
+ auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
+ auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
+ unsigned MulOpc, AccOpc;
+ switch (IntNo) {
+ default:
+ llvm_unreachable("Unexpected RV32 horizontal multiply intrinsic");
+ case Intrinsic::riscv_pm4add:
+ MulOpc = RISCVISD::PM2WADD;
+ AccOpc = RISCVISD::PM2WADDA;
+ break;
+ case Intrinsic::riscv_pm4addu:
+ MulOpc = RISCVISD::PM2WADDU;
+ AccOpc = RISCVISD::PM2WADDAU;
+ break;
+ case Intrinsic::riscv_pm4addsu:
+ MulOpc = RISCVISD::PM2WADDSU;
+ AccOpc = RISCVISD::PM2WADDASU;
+ break;
+ }
+ SDValue Acc = DAG.getNode(MulOpc, DL, MVT::v2i32, Rs1Lo, Rs2Lo);
+ return DAG.getNode(AccOpc, DL, MVT::v2i32, Acc, Rs1Hi, Rs2Hi);
+ }
+
+ assert(Rs1.getSimpleValueType() == MVT::v2i32 &&
+ "Unexpected RV32 horizontal multiply source type");
+ auto Extract = [&](SDValue V, unsigned Idx) {
+ return DAG.getExtractVectorElt(DL, MVT::i32, V, Idx);
+ };
+ SDValue Rs1Lo = Extract(Rs1, 0);
+ SDValue Rs1Hi = Extract(Rs1, 1);
+ SDValue Rs2Lo = Extract(Rs2, 0);
+ SDValue Rs2Hi = Extract(Rs2, 1);
+
+ if (IntNo == Intrinsic::riscv_pmq2add || IntNo == Intrinsic::riscv_pmqr2add) {
+ unsigned AccOpc = IntNo == Intrinsic::riscv_pmq2add ? RISCVISD::MQWACC
+ : RISCVISD::MQRWACC;
+ SDValue Acc = DAG.getConstant(0, DL, MVT::v2i32);
+ Acc = DAG.getNode(AccOpc, DL, MVT::v2i32, Acc, Rs1Lo, Rs2Lo);
+ return DAG.getNode(AccOpc, DL, MVT::v2i32, Acc, Rs1Hi, Rs2Hi);
+ }
+
+ unsigned MulOpc = ISD::SMUL_LOHI;
+ if (IntNo == Intrinsic::riscv_pm2addu)
+ MulOpc = ISD::UMUL_LOHI;
+ else if (IntNo == Intrinsic::riscv_pm2addsu)
+ MulOpc = RISCVISD::WMULSU;
+ bool IsSub =
+ IntNo == Intrinsic::riscv_pm2sub || IntNo == Intrinsic::riscv_pm2sub_x;
+ if (IntNo == Intrinsic::riscv_pm2add_x || IntNo == Intrinsic::riscv_pm2sub_x)
+ std::swap(Rs2Lo, Rs2Hi);
+
+ SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32);
+ SDValue LoMul = DAG.getNode(MulOpc, DL, VTs, Rs1Lo, Rs2Lo);
+ SDValue HiMul = DAG.getNode(MulOpc, DL, VTs, Rs1Hi, Rs2Hi);
+ unsigned Opc = IsSub ? RISCVISD::SUBD : RISCVISD::ADDD;
+ SDValue Res = DAG.getNode(Opc, DL, VTs, LoMul, LoMul.getValue(1), HiMul,
+ HiMul.getValue(1));
+ return DAG.getBuildVector(MVT::v2i32, DL, {Res, Res.getValue(1)});
+}
+
SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
SelectionDAG &DAG) const {
unsigned IntNo = Op.getConstantOperandVal(0);
@@ -16952,8 +17016,8 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
SDValue Rs2 = N->getOperand(2);
if (!Subtarget.is64Bit() && VT == MVT::i64) {
- SDValue Pair = DAG.getNode(Opc, DL, MVT::v2i32, Rs1, Rs2);
- Results.push_back(DAG.getBitcast(MVT::i64, Pair));
+ SDValue Res = lowerRV32HorizontalMul64(IntNo, Rs1, Rs2, DL, DAG);
+ Results.push_back(DAG.getBitcast(MVT::i64, Res));
return;
}
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
index 807792928a336..38b0b11397276 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
@@ -1879,6 +1879,28 @@ def riscv_pmulsu_halves_00
def riscv_pmulsu_halves_11
: RVSDNode<"PMULSU_HALVES_11", SDT_RISCVPackedWideningMulByHalves>;
+// Packed multiplication with widening horizontal addition.
+def SDT_RISCVPackedWideningHorizontalAdd
+ : SDTypeProfile<1, 2, [SDTCisVT<0, v2i32>, SDTCisVT<1, v2i16>,
+ SDTCisSameAs<1, 2>]>;
+def riscv_pm2wadd
+ : RVSDNode<"PM2WADD", SDT_RISCVPackedWideningHorizontalAdd>;
+def riscv_pm2waddu
+ : RVSDNode<"PM2WADDU", SDT_RISCVPackedWideningHorizontalAdd>;
+def riscv_pm2waddsu
+ : RVSDNode<"PM2WADDSU", SDT_RISCVPackedWideningHorizontalAdd>;
+
+// Packed multiplication with widening horizontal addition and accumulate.
+def SDT_RISCVPackedWideningHorizontalAddAcc
+ : SDTypeProfile<1, 3, [SDTCisVT<0, v2i32>, SDTCisSameAs<0, 1>,
+ SDTCisVT<2, v2i16>, SDTCisSameAs<2, 3>]>;
+def riscv_pm2wadda
+ : RVSDNode<"PM2WADDA", SDT_RISCVPackedWideningHorizontalAddAcc>;
+def riscv_pm2waddau
+ : RVSDNode<"PM2WADDAU", SDT_RISCVPackedWideningHorizontalAddAcc>;
+def riscv_pm2waddasu
+ : RVSDNode<"PM2WADDASU", SDT_RISCVPackedWideningHorizontalAddAcc>;
+
def SDT_RISCVWideningShiftLeft : SDTypeProfile<2, 2, [SDTCisVT<0, i32>,
SDTCisSameAs<0, 1>,
SDTCisSameAs<0, 2>,
@@ -1956,6 +1978,13 @@ def riscv_pm2addsu : RVSDNode<"PM2ADDSU", SDT_RISCVHorizontalMul>;
def riscv_mulq : RVSDNode<"MULQ", SDTIntBinOp, [SDNPCommutative]>;
def riscv_mulqr : RVSDNode<"MULQR", SDTIntBinOp, [SDNPCommutative]>;
+// "Q-format" multiply with widening accumulate.
+def SDT_RISCVQWideningMulAcc
+ : SDTypeProfile<1, 3, [SDTCisVT<0, v2i32>, SDTCisSameAs<0, 1>,
+ SDTCisVT<2, i32>, SDTCisSameAs<2, 3>]>;
+def riscv_mqwacc : RVSDNode<"MQWACC", SDT_RISCVQWideningMulAcc>;
+def riscv_mqrwacc : RVSDNode<"MQRWACC", SDT_RISCVQWideningMulAcc>;
+
def SDT_RISCVPackedShift : SDTypeProfile<1, 2, [SDTCisVec<0>,
SDTCisSameAs<0, 1>,
SDTCisVT<2, XLenVT>]>;
@@ -2362,6 +2391,12 @@ let append Predicates = [IsRV32] in {
def : PatGprGpr<riscv_mulq, MULQ, i32>;
def : PatGprGpr<riscv_mulqr, MULQR, i32>;
+ // "Q-format" multiply with widening accumulate patterns.
+ def : Pat<(v2i32 (riscv_mqwacc (v2i32 GPRPair:$rd), GPR:$rs1, GPR:$rs2)),
+ (MQWACC GPRPair:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(v2i32 (riscv_mqrwacc (v2i32 GPRPair:$rd), GPR:$rs1, GPR:$rs2)),
+ (MQRWACC GPRPair:$rd, GPR:$rs1, GPR:$rs2)>;
+
// Packed multiplication with horizontal addition patterns.
def : PatHorizontalMul<riscv_pm4add, PM4ADD_B, i32, v4i8>;
def : PatHorizontalMul<riscv_pm2add, PM2ADD_H, i32, v2i16>;
@@ -2377,82 +2412,6 @@ let append Predicates = [IsRV32] in {
def : PatHorizontalMul<riscv_pm4addsu, PM4ADDSU_B, i32, v4i8>;
def : PatHorizontalMul<riscv_pm2addsu, PM2ADDSU_H, i32, v2i16>;
- // Select the RV32 v2i32 result produced by type legalization into the
- // instruction sequences specified for the 64-bit intrinsics.
- def : Pat<(v2i32 (riscv_pm2add (v2i32 GPRPair:$rs1),
- (v2i32 GPRPair:$rs2))),
- (WMACC (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)),
- (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>;
- def : Pat<(v2i32 (riscv_pm2add_x (v2i32 GPRPair:$rs1),
- (v2i32 GPRPair:$rs2))),
- (WMACC (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd)),
- (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even))>;
- def : Pat<(v2i32 (riscv_pm2addu (v2i32 GPRPair:$rs1),
- (v2i32 GPRPair:$rs2))),
- (WMACCU (WMULU (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)),
- (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>;
- def : Pat<(v2i32 (riscv_pmq2add (v2i32 GPRPair:$rs1),
- (v2i32 GPRPair:$rs2))),
- (MQWACC (v2i32 (MQWACC
- (v2i32 (BuildGPRPair (i32 X0), (i32 X0))),
- (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even))),
- (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>;
- def : Pat<(v2i32 (riscv_pmqr2add (v2i32 GPRPair:$rs1),
- (v2i32 GPRPair:$rs2))),
- (MQRWACC (v2i32 (MQRWACC
- (v2i32 (BuildGPRPair (i32 X0), (i32 X0))),
- (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even))),
- (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>;
- def : Pat<(v2i32 (riscv_pm2sub (v2i32 GPRPair:$rs1),
- (v2i32 GPRPair:$rs2))),
- (SUBD (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)),
- (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd)))>;
- def : Pat<(v2i32 (riscv_pm2sub_x (v2i32 GPRPair:$rs1),
- (v2i32 GPRPair:$rs2))),
- (SUBD (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd)),
- (WMUL (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)))>;
- def : Pat<(v2i32 (riscv_pm2addsu (v2i32 GPRPair:$rs1),
- (v2i32 GPRPair:$rs2))),
- (WMACCSU (WMULSU (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)),
- (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>;
- def : Pat<(v2i32 (riscv_pm4add (v4i16 GPRPair:$rs1),
- (v4i16 GPRPair:$rs2))),
- (PM2WADDA_H
- (PM2WADD_H (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)),
- (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>;
- def : Pat<(v2i32 (riscv_pm4addu (v4i16 GPRPair:$rs1),
- (v4i16 GPRPair:$rs2))),
- (PM2WADDAU_H
- (PM2WADDU_H (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)),
- (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>;
- def : Pat<(v2i32 (riscv_pm4addsu (v4i16 GPRPair:$rs1),
- (v4i16 GPRPair:$rs2))),
- (PM2WADDASU_H
- (PM2WADDSU_H (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_even),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_even)),
- (EXTRACT_SUBREG GPRPair:$rs1, sub_gpr_odd),
- (EXTRACT_SUBREG GPRPair:$rs2, sub_gpr_odd))>;
-
// Halfword multiply patterns where one operand is a sext.h or zext.h and
// the other is a sext.h or zext.h or is known to be sign/zero-extended. We
// prefer plain mul when both operands are known to be sign/zero-extended.
@@ -2518,6 +2477,26 @@ let append Predicates = [IsRV32] in {
def : Pat<(v2i32 (riscv_pwmulsu (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
(PWMULSU_H GPR:$rs1, GPR:$rs2)>;
+ // Packed multiplication with widening horizontal addition patterns.
+ def : Pat<(v2i32 (riscv_pm2wadd (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
+ (PM2WADD_H GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(v2i32 (riscv_pm2waddu (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
+ (PM2WADDU_H GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(v2i32 (riscv_pm2waddsu (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
+ (PM2WADDSU_H GPR:$rs1, GPR:$rs2)>;
+
+ // Packed multiplication with widening horizontal addition and accumulate
+ // patterns.
+ def : Pat<(v2i32 (riscv_pm2wadda (v2i32 GPRPair:$rd),
+ (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
+ (PM2WADDA_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(v2i32 (riscv_pm2waddau (v2i32 GPRPair:$rd),
+ (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
+ (PM2WADDAU_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(v2i32 (riscv_pm2waddasu (v2i32 GPRPair:$rd),
+ (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
+ (PM2WADDASU_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>;
+
// 8/16-bit bitreverse patterns
// With Zbkb, brev8 reverses the bits within each byte directly; otherwise
// reverse all bits then swap the bytes back.
diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
index ad2d4e2dbeb11..4d9e7acd55e0d 100644
--- a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
+++ b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
@@ -7312,9 +7312,9 @@ define <2 x i32> @test_pm2addsu_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) {
define i64 @test_pm2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
; RV32-LABEL: test_pm2add_v2i32_i64:
; RV32: # %bb.0:
-; RV32-NEXT: mv a4, a1
-; RV32-NEXT: wmul a0, a0, a2
-; RV32-NEXT: wmacc a0, a4, a3
+; RV32-NEXT: mv a4, a0
+; RV32-NEXT: wmul a0, a1, a3
+; RV32-NEXT: wmacc a0, a4, a2
; RV32-NEXT: ret
;
; RV64-LABEL: test_pm2add_v2i32_i64:
@@ -7328,9 +7328,9 @@ define i64 @test_pm2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
define i64 @test_pm2add_x_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
; RV32-LABEL: test_pm2add_x_v2i32_i64:
; RV32: # %bb.0:
-; RV32-NEXT: mv a4, a1
-; RV32-NEXT: wmul a0, a0, a3
-; RV32-NEXT: wmacc a0, a4, a2
+; RV32-NEXT: mv a4, a0
+; RV32-NEXT: wmul a0, a1, a2
+; RV32-NEXT: wmacc a0, a4, a3
; RV32-NEXT: ret
;
; RV64-LABEL: test_pm2add_x_v2i32_i64:
@@ -7344,9 +7344,9 @@ define i64 @test_pm2add_x_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
define i64 @test_pm2addu_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
; RV32-LABEL: test_pm2addu_v2i32_i64:
; RV32: # %bb.0:
-; RV32-NEXT: mv a4, a1
-; RV32-NEXT: wmulu a0, a0, a2
-; RV32-NEXT: wmaccu a0, a4, a3
+; RV32-NEXT: mv a4, a0
+; RV32-NEXT: wmulu a0, a1, a3
+; RV32-NEXT: wmaccu a0, a4, a2
; RV32-NEXT: ret
;
; RV64-LABEL: test_pm2addu_v2i32_i64:
@@ -7360,8 +7360,7 @@ define i64 @test_pm2addu_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
define i64 @test_pmq2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
; RV32-LABEL: test_pmq2add_v2i32_i64:
; RV32: # %bb.0:
-; RV32-NEXT: li a4, 0
-; RV32-NEXT: li a5, 0
+; RV32-NEXT: mvd a4, zero
; RV32-NEXT: mqwacc a4, a0, a2
; RV32-NEXT: mqwacc a4, a1, a3
; RV32-NEXT: mvd a0, a4
@@ -7378,8 +7377,7 @@ define i64 @test_pmq2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
define i64 @test_pmqr2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
; RV32-LABEL: test_pmqr2add_v2i32_i64:
; RV32: # %bb.0:
-; RV32-NEXT: li a4, 0
-; RV32-NEXT: li a5, 0
+; RV32-NEXT: mvd a4, zero
; RV32-NEXT: mqrwacc a4, a0, a2
; RV32-NEXT: mqrwacc a4, a1, a3
; RV32-NEXT: mvd a0, a4
@@ -7428,9 +7426,9 @@ define i64 @test_pm2sub_x_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
define i64 @test_pm2addsu_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
; RV32-LABEL: test_pm2addsu_v2i32_i64:
; RV32: # %bb.0:
-; RV32-NEXT: mv a4, a1
-; RV32-NEXT: wmulsu a0, a0, a2
-; RV32-NEXT: wmaccsu a0, a4, a3
+; RV32-NEXT: mv a4, a0
+; RV32-NEXT: wmulsu a0, a1, a3
+; RV32-NEXT: wmaccsu a0, a4, a2
; RV32-NEXT: ret
;
; RV64-LABEL: test_pm2addsu_v2i32_i64:
>From 7dd47a4f6c0224a692f69d023f0e37790a299401 Mon Sep 17 00:00:00 2001
From: XChy <xxs_chy at outlook.com>
Date: Thu, 27 Aug 2026 02:09:22 +0800
Subject: [PATCH 4/4] use build_pair
---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 22 ++++++++++++---------
llvm/lib/Target/RISCV/RISCVInstrInfoP.td | 13 ------------
llvm/test/CodeGen/RISCV/rvp-simd-64.ll | 6 ++++--
3 files changed, 17 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index b64998d98a913..622dbce1f3fa3 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -12365,6 +12365,10 @@ static unsigned getRVPHorizontalMulOpcode(unsigned IntNo) {
static SDValue lowerRV32HorizontalMul64(unsigned IntNo, SDValue Rs1,
SDValue Rs2, const SDLoc &DL,
SelectionDAG &DAG) {
+ auto Extract = [&](SDValue V, unsigned Idx) {
+ return DAG.getExtractVectorElt(DL, MVT::i32, V, Idx);
+ };
+
if (Rs1.getSimpleValueType() == MVT::v4i16) {
auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
@@ -12386,14 +12390,12 @@ static SDValue lowerRV32HorizontalMul64(unsigned IntNo, SDValue Rs1,
break;
}
SDValue Acc = DAG.getNode(MulOpc, DL, MVT::v2i32, Rs1Lo, Rs2Lo);
- return DAG.getNode(AccOpc, DL, MVT::v2i32, Acc, Rs1Hi, Rs2Hi);
+ Acc = DAG.getNode(AccOpc, DL, MVT::v2i32, Acc, Rs1Hi, Rs2Hi);
+ return DAG.getBitcast(MVT::i64, Acc);
}
assert(Rs1.getSimpleValueType() == MVT::v2i32 &&
"Unexpected RV32 horizontal multiply source type");
- auto Extract = [&](SDValue V, unsigned Idx) {
- return DAG.getExtractVectorElt(DL, MVT::i32, V, Idx);
- };
SDValue Rs1Lo = Extract(Rs1, 0);
SDValue Rs1Hi = Extract(Rs1, 1);
SDValue Rs2Lo = Extract(Rs2, 0);
@@ -12402,9 +12404,11 @@ static SDValue lowerRV32HorizontalMul64(unsigned IntNo, SDValue Rs1,
if (IntNo == Intrinsic::riscv_pmq2add || IntNo == Intrinsic::riscv_pmqr2add) {
unsigned AccOpc = IntNo == Intrinsic::riscv_pmq2add ? RISCVISD::MQWACC
: RISCVISD::MQRWACC;
- SDValue Acc = DAG.getConstant(0, DL, MVT::v2i32);
- Acc = DAG.getNode(AccOpc, DL, MVT::v2i32, Acc, Rs1Lo, Rs2Lo);
- return DAG.getNode(AccOpc, DL, MVT::v2i32, Acc, Rs1Hi, Rs2Hi);
+ SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32);
+ SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
+ SDValue Acc = DAG.getNode(AccOpc, DL, VTs, {Zero, Zero, Rs1Lo, Rs2Lo});
+ Acc = DAG.getNode(AccOpc, DL, VTs, {Acc, Acc.getValue(1), Rs1Hi, Rs2Hi});
+ return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Acc, Acc.getValue(1));
}
unsigned MulOpc = ISD::SMUL_LOHI;
@@ -12423,7 +12427,7 @@ static SDValue lowerRV32HorizontalMul64(unsigned IntNo, SDValue Rs1,
unsigned Opc = IsSub ? RISCVISD::SUBD : RISCVISD::ADDD;
SDValue Res = DAG.getNode(Opc, DL, VTs, LoMul, LoMul.getValue(1), HiMul,
HiMul.getValue(1));
- return DAG.getBuildVector(MVT::v2i32, DL, {Res, Res.getValue(1)});
+ return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Res, Res.getValue(1));
}
SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
@@ -17017,7 +17021,7 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
if (!Subtarget.is64Bit() && VT == MVT::i64) {
SDValue Res = lowerRV32HorizontalMul64(IntNo, Rs1, Rs2, DL, DAG);
- Results.push_back(DAG.getBitcast(MVT::i64, Res));
+ Results.push_back(Res);
return;
}
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
index 38b0b11397276..2a180ceae66e8 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
@@ -1978,13 +1978,6 @@ def riscv_pm2addsu : RVSDNode<"PM2ADDSU", SDT_RISCVHorizontalMul>;
def riscv_mulq : RVSDNode<"MULQ", SDTIntBinOp, [SDNPCommutative]>;
def riscv_mulqr : RVSDNode<"MULQR", SDTIntBinOp, [SDNPCommutative]>;
-// "Q-format" multiply with widening accumulate.
-def SDT_RISCVQWideningMulAcc
- : SDTypeProfile<1, 3, [SDTCisVT<0, v2i32>, SDTCisSameAs<0, 1>,
- SDTCisVT<2, i32>, SDTCisSameAs<2, 3>]>;
-def riscv_mqwacc : RVSDNode<"MQWACC", SDT_RISCVQWideningMulAcc>;
-def riscv_mqrwacc : RVSDNode<"MQRWACC", SDT_RISCVQWideningMulAcc>;
-
def SDT_RISCVPackedShift : SDTypeProfile<1, 2, [SDTCisVec<0>,
SDTCisSameAs<0, 1>,
SDTCisVT<2, XLenVT>]>;
@@ -2391,12 +2384,6 @@ let append Predicates = [IsRV32] in {
def : PatGprGpr<riscv_mulq, MULQ, i32>;
def : PatGprGpr<riscv_mulqr, MULQR, i32>;
- // "Q-format" multiply with widening accumulate patterns.
- def : Pat<(v2i32 (riscv_mqwacc (v2i32 GPRPair:$rd), GPR:$rs1, GPR:$rs2)),
- (MQWACC GPRPair:$rd, GPR:$rs1, GPR:$rs2)>;
- def : Pat<(v2i32 (riscv_mqrwacc (v2i32 GPRPair:$rd), GPR:$rs1, GPR:$rs2)),
- (MQRWACC GPRPair:$rd, GPR:$rs1, GPR:$rs2)>;
-
// Packed multiplication with horizontal addition patterns.
def : PatHorizontalMul<riscv_pm4add, PM4ADD_B, i32, v4i8>;
def : PatHorizontalMul<riscv_pm2add, PM2ADD_H, i32, v2i16>;
diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
index 4d9e7acd55e0d..93834337888f1 100644
--- a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
+++ b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
@@ -7360,7 +7360,8 @@ define i64 @test_pm2addu_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
define i64 @test_pmq2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
; RV32-LABEL: test_pmq2add_v2i32_i64:
; RV32: # %bb.0:
-; RV32-NEXT: mvd a4, zero
+; RV32-NEXT: li a4, 0
+; RV32-NEXT: li a5, 0
; RV32-NEXT: mqwacc a4, a0, a2
; RV32-NEXT: mqwacc a4, a1, a3
; RV32-NEXT: mvd a0, a4
@@ -7377,7 +7378,8 @@ define i64 @test_pmq2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
define i64 @test_pmqr2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
; RV32-LABEL: test_pmqr2add_v2i32_i64:
; RV32: # %bb.0:
-; RV32-NEXT: mvd a4, zero
+; RV32-NEXT: li a4, 0
+; RV32-NEXT: li a5, 0
; RV32-NEXT: mqrwacc a4, a0, a2
; RV32-NEXT: mqrwacc a4, a1, a3
; RV32-NEXT: mvd a0, a4
More information about the llvm-commits
mailing list