[llvm] d326c16 - [RISCV] Support Packed Multiplication with Horizontal Addition (#218430)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 1 04:08:02 PDT 2026
Author: Hongyu Chen
Date: 2026-09-01T19:07:57+08:00
New Revision: d326c1654543f5a3de2f3ff5639f95e3d71eb7d3
URL: https://github.com/llvm/llvm-project/commit/d326c1654543f5a3de2f3ff5639f95e3d71eb7d3
DIFF: https://github.com/llvm/llvm-project/commit/d326c1654543f5a3de2f3ff5639f95e3d71eb7d3.diff
LOG: [RISCV] Support Packed Multiplication with Horizontal Addition (#218430)
See also
https://github.com/riscv/riscv-p-spec/blob/master/P-ext-intrinsics.adoc#packed-multiplication-with-horizontal-addition.
Added:
Modified:
clang/include/clang/Basic/BuiltinsRISCV.td
clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
clang/lib/Headers/riscv_packed_simd.h
cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
llvm/include/llvm/IR/IntrinsicsRISCV.td
llvm/lib/Target/RISCV/RISCVISelLowering.cpp
llvm/lib/Target/RISCV/RISCVInstrInfoP.td
llvm/test/CodeGen/RISCV/rvp-simd-32.ll
llvm/test/CodeGen/RISCV/rvp-simd-64.ll
Removed:
################################################################################
diff --git a/clang/include/clang/Basic/BuiltinsRISCV.td b/clang/include/clang/Basic/BuiltinsRISCV.td
index b5aa5861d3dd2..58184479d3c27 100644
--- a/clang/include/clang/Basic/BuiltinsRISCV.td
+++ b/clang/include/clang/Basic/BuiltinsRISCV.td
@@ -291,6 +291,47 @@ def pmhaccsu_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<2, i
def pmhraccsu_i16x4 : RISCVBuiltin<"_Vector<4, short>(_Vector<4, short>, _Vector<4, short>, _Vector<4, unsigned short>)">;
def pmhraccsu_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<2, int>, _Vector<2, unsigned int>)">;
+// Packed Multiplication with Horizontal Addition (32-bit)
+def pm4add_i8x4 : RISCVBuiltin<"int(_Vector<4, signed char>, _Vector<4, signed char>)">;
+def pm2add_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">;
+def pm2add_x_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">;
+def pm4addu_u8x4 : RISCVBuiltin<"unsigned int(_Vector<4, unsigned char>, _Vector<4, unsigned char>)">;
+def pm2addu_u16x2 : RISCVBuiltin<"unsigned int(_Vector<2, unsigned short>, _Vector<2, unsigned short>)">;
+def pmq2add_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">;
+def pmqr2add_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">;
+def pm2sadd_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">;
+def pm2sadd_x_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">;
+def pm2sub_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">;
+def pm2sub_x_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, short>)">;
+def pm4addsu_i8x4 : RISCVBuiltin<"int(_Vector<4, signed char>, _Vector<4, unsigned char>)">;
+def pm2addsu_i16x2 : RISCVBuiltin<"int(_Vector<2, short>, _Vector<2, unsigned short>)">;
+
+// Packed Multiplication with Horizontal Addition (64-bit)
+def pm4add_i8x8 : RISCVBuiltin<"_Vector<2, int>(_Vector<8, signed char>, _Vector<8, signed char>)">;
+def pm2add_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">;
+def pm2add_x_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">;
+def pm4addu_u8x8 : RISCVBuiltin<"_Vector<2, unsigned int>(_Vector<8, unsigned char>, _Vector<8, unsigned char>)">;
+def pm2addu_u16x4 : RISCVBuiltin<"_Vector<2, unsigned int>(_Vector<4, unsigned short>, _Vector<4, unsigned short>)">;
+def pmq2add_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">;
+def pmqr2add_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">;
+def pm2sadd_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">;
+def pm2sadd_x_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">;
+def pm2sub_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">;
+def pm2sub_x_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, short>)">;
+def pm4addsu_i8x8 : RISCVBuiltin<"_Vector<2, int>(_Vector<8, signed char>, _Vector<8, unsigned char>)">;
+def pm2addsu_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<4, short>, _Vector<4, unsigned short>)">;
+def pm2add_i32x2 : RISCVBuiltin<"int64_t(_Vector<2, int>, _Vector<2, int>)">;
+def pm2add_x_i32x2 : RISCVBuiltin<"int64_t(_Vector<2, int>, _Vector<2, int>)">;
+def pm2addu_u32x2 : RISCVBuiltin<"uint64_t(_Vector<2, unsigned int>, _Vector<2, unsigned int>)">;
+def pmq2add_i32x2 : RISCVBuiltin<"int64_t(_Vector<2, int>, _Vector<2, int>)">;
+def pm2sub_i32x2 : RISCVBuiltin<"int64_t(_Vector<2, int>, _Vector<2, int>)">;
+def pm2sub_x_i32x2 : RISCVBuiltin<"int64_t(_Vector<2, int>, _Vector<2, int>)">;
+def pm2addsu_i32x2 : RISCVBuiltin<"int64_t(_Vector<2, int>, _Vector<2, unsigned int>)">;
+def pmqr2add_i32x2 : RISCVBuiltin<"int64_t(_Vector<2, int>, _Vector<2, int>)">;
+def pm4add_i16x4 : RISCVBuiltin<"int64_t(_Vector<4, short>, _Vector<4, short>)">;
+def pm4addu_u16x4 : RISCVBuiltin<"uint64_t(_Vector<4, unsigned short>, _Vector<4, unsigned short>)">;
+def pm4addsu_i16x4 : RISCVBuiltin<"int64_t(_Vector<4, short>, _Vector<4, unsigned short>)">;
+
// Packed Absolute Difference Sum (32-bit)
def pabdsumu_u8x4_u32 : RISCVBuiltin<"unsigned int(_Vector<4, unsigned char>, _Vector<4, unsigned char>)">;
def pabdsumau_u8x4_u32 : RISCVBuiltin<"unsigned int(unsigned int, _Vector<4, unsigned char>, _Vector<4, unsigned char>)">;
diff --git a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
index 66e04f2ea06b3..2e9fbf771bed1 100644
--- a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
@@ -1531,6 +1531,116 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID,
break;
}
+ // Packed Multiplication with Horizontal Addition
+ case RISCV::BI__builtin_riscv_pm4add_i8x4:
+ case RISCV::BI__builtin_riscv_pm4add_i8x8:
+ case RISCV::BI__builtin_riscv_pm4add_i16x4:
+ case RISCV::BI__builtin_riscv_pm2add_i16x2:
+ case RISCV::BI__builtin_riscv_pm2add_i16x4:
+ case RISCV::BI__builtin_riscv_pm2add_i32x2:
+ case RISCV::BI__builtin_riscv_pm2add_x_i16x2:
+ case RISCV::BI__builtin_riscv_pm2add_x_i16x4:
+ case RISCV::BI__builtin_riscv_pm2add_x_i32x2:
+ case RISCV::BI__builtin_riscv_pm4addu_u8x4:
+ case RISCV::BI__builtin_riscv_pm4addu_u8x8:
+ case RISCV::BI__builtin_riscv_pm4addu_u16x4:
+ case RISCV::BI__builtin_riscv_pm2addu_u16x2:
+ case RISCV::BI__builtin_riscv_pm2addu_u16x4:
+ case RISCV::BI__builtin_riscv_pm2addu_u32x2:
+ case RISCV::BI__builtin_riscv_pmq2add_i16x2:
+ case RISCV::BI__builtin_riscv_pmq2add_i16x4:
+ case RISCV::BI__builtin_riscv_pmq2add_i32x2:
+ case RISCV::BI__builtin_riscv_pmqr2add_i16x2:
+ case RISCV::BI__builtin_riscv_pmqr2add_i16x4:
+ case RISCV::BI__builtin_riscv_pmqr2add_i32x2:
+ case RISCV::BI__builtin_riscv_pm2sadd_i16x2:
+ case RISCV::BI__builtin_riscv_pm2sadd_i16x4:
+ case RISCV::BI__builtin_riscv_pm2sadd_x_i16x2:
+ case RISCV::BI__builtin_riscv_pm2sadd_x_i16x4:
+ case RISCV::BI__builtin_riscv_pm2sub_i16x2:
+ case RISCV::BI__builtin_riscv_pm2sub_i16x4:
+ case RISCV::BI__builtin_riscv_pm2sub_i32x2:
+ case RISCV::BI__builtin_riscv_pm2sub_x_i16x2:
+ case RISCV::BI__builtin_riscv_pm2sub_x_i16x4:
+ case RISCV::BI__builtin_riscv_pm2sub_x_i32x2:
+ case RISCV::BI__builtin_riscv_pm4addsu_i8x4:
+ case RISCV::BI__builtin_riscv_pm4addsu_i8x8:
+ case RISCV::BI__builtin_riscv_pm4addsu_i16x4:
+ case RISCV::BI__builtin_riscv_pm2addsu_i16x2:
+ case RISCV::BI__builtin_riscv_pm2addsu_i16x4:
+ case RISCV::BI__builtin_riscv_pm2addsu_i32x2: {
+ switch (BuiltinID) {
+ default:
+ llvm_unreachable("unexpected builtin ID");
+ case RISCV::BI__builtin_riscv_pm4add_i8x4:
+ case RISCV::BI__builtin_riscv_pm4add_i8x8:
+ case RISCV::BI__builtin_riscv_pm4add_i16x4:
+ ID = Intrinsic::riscv_pm4add;
+ break;
+ case RISCV::BI__builtin_riscv_pm2add_i16x2:
+ case RISCV::BI__builtin_riscv_pm2add_i16x4:
+ case RISCV::BI__builtin_riscv_pm2add_i32x2:
+ ID = Intrinsic::riscv_pm2add;
+ break;
+ case RISCV::BI__builtin_riscv_pm2add_x_i16x2:
+ case RISCV::BI__builtin_riscv_pm2add_x_i16x4:
+ case RISCV::BI__builtin_riscv_pm2add_x_i32x2:
+ ID = Intrinsic::riscv_pm2add_x;
+ break;
+ case RISCV::BI__builtin_riscv_pm4addu_u8x4:
+ case RISCV::BI__builtin_riscv_pm4addu_u8x8:
+ case RISCV::BI__builtin_riscv_pm4addu_u16x4:
+ ID = Intrinsic::riscv_pm4addu;
+ break;
+ case RISCV::BI__builtin_riscv_pm2addu_u16x2:
+ case RISCV::BI__builtin_riscv_pm2addu_u16x4:
+ case RISCV::BI__builtin_riscv_pm2addu_u32x2:
+ ID = Intrinsic::riscv_pm2addu;
+ break;
+ case RISCV::BI__builtin_riscv_pmq2add_i16x2:
+ case RISCV::BI__builtin_riscv_pmq2add_i16x4:
+ case RISCV::BI__builtin_riscv_pmq2add_i32x2:
+ ID = Intrinsic::riscv_pmq2add;
+ break;
+ case RISCV::BI__builtin_riscv_pmqr2add_i16x2:
+ case RISCV::BI__builtin_riscv_pmqr2add_i16x4:
+ case RISCV::BI__builtin_riscv_pmqr2add_i32x2:
+ ID = Intrinsic::riscv_pmqr2add;
+ break;
+ case RISCV::BI__builtin_riscv_pm2sadd_i16x2:
+ case RISCV::BI__builtin_riscv_pm2sadd_i16x4:
+ ID = Intrinsic::riscv_pm2sadd;
+ break;
+ case RISCV::BI__builtin_riscv_pm2sadd_x_i16x2:
+ case RISCV::BI__builtin_riscv_pm2sadd_x_i16x4:
+ ID = Intrinsic::riscv_pm2sadd_x;
+ break;
+ case RISCV::BI__builtin_riscv_pm2sub_i16x2:
+ case RISCV::BI__builtin_riscv_pm2sub_i16x4:
+ case RISCV::BI__builtin_riscv_pm2sub_i32x2:
+ ID = Intrinsic::riscv_pm2sub;
+ break;
+ case RISCV::BI__builtin_riscv_pm2sub_x_i16x2:
+ case RISCV::BI__builtin_riscv_pm2sub_x_i16x4:
+ case RISCV::BI__builtin_riscv_pm2sub_x_i32x2:
+ ID = Intrinsic::riscv_pm2sub_x;
+ break;
+ case RISCV::BI__builtin_riscv_pm4addsu_i8x4:
+ case RISCV::BI__builtin_riscv_pm4addsu_i8x8:
+ case RISCV::BI__builtin_riscv_pm4addsu_i16x4:
+ ID = Intrinsic::riscv_pm4addsu;
+ break;
+ case RISCV::BI__builtin_riscv_pm2addsu_i16x2:
+ case RISCV::BI__builtin_riscv_pm2addsu_i16x4:
+ case RISCV::BI__builtin_riscv_pm2addsu_i32x2:
+ ID = Intrinsic::riscv_pm2addsu;
+ break;
+ }
+
+ IntrinsicTypes = {ResultType, Ops[0]->getType()};
+ break;
+ }
+
// Packed Reduction Sum
case RISCV::BI__builtin_riscv_predsum_i8x4_i32:
case RISCV::BI__builtin_riscv_predsum_i16x2_i32:
diff --git a/clang/lib/Headers/riscv_packed_simd.h b/clang/lib/Headers/riscv_packed_simd.h
index 1bae9ac9cec37..bb0e7624a50e9 100644
--- a/clang/lib/Headers/riscv_packed_simd.h
+++ b/clang/lib/Headers/riscv_packed_simd.h
@@ -829,6 +829,47 @@ __packed_ternary_builtin_mixed(pmhaccsu_i32x2, int32x2_t, int32x2_t, uint32x2_t,
__packed_ternary_builtin_mixed(pmhraccsu_i16x4, int16x4_t, int16x4_t, uint16x4_t, __builtin_riscv_pmhraccsu_i16x4)
__packed_ternary_builtin_mixed(pmhraccsu_i32x2, int32x2_t, int32x2_t, uint32x2_t, __builtin_riscv_pmhraccsu_i32x2)
+/* Packed Multiplication with Horizontal Addition (32-bit) */
+__packed_binary_builtin_mixed(pm4add_i8x4, int32_t, int8x4_t, int8x4_t, __builtin_riscv_pm4add_i8x4)
+__packed_binary_builtin_mixed(pm2add_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2add_i16x2)
+__packed_binary_builtin_mixed(pm2add_x_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2add_x_i16x2)
+__packed_binary_builtin_mixed(pm4addu_u8x4, uint32_t, uint8x4_t, uint8x4_t, __builtin_riscv_pm4addu_u8x4)
+__packed_binary_builtin_mixed(pm2addu_u16x2, uint32_t, uint16x2_t, uint16x2_t, __builtin_riscv_pm2addu_u16x2)
+__packed_binary_builtin_mixed(pmq2add_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pmq2add_i16x2)
+__packed_binary_builtin_mixed(pmqr2add_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pmqr2add_i16x2)
+__packed_binary_builtin_mixed(pm2sadd_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2sadd_i16x2)
+__packed_binary_builtin_mixed(pm2sadd_x_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2sadd_x_i16x2)
+__packed_binary_builtin_mixed(pm2sub_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2sub_i16x2)
+__packed_binary_builtin_mixed(pm2sub_x_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2sub_x_i16x2)
+__packed_binary_builtin_mixed(pm4addsu_i8x4, int32_t, int8x4_t, uint8x4_t, __builtin_riscv_pm4addsu_i8x4)
+__packed_binary_builtin_mixed(pm2addsu_i16x2, int32_t, int16x2_t, uint16x2_t, __builtin_riscv_pm2addsu_i16x2)
+
+/* Packed Multiplication with Horizontal Addition (64-bit) */
+__packed_binary_builtin_mixed(pm4add_i8x8, int32x2_t, int8x8_t, int8x8_t, __builtin_riscv_pm4add_i8x8)
+__packed_binary_builtin_mixed(pm2add_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2add_i16x4)
+__packed_binary_builtin_mixed(pm2add_x_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2add_x_i16x4)
+__packed_binary_builtin_mixed(pm4addu_u8x8, uint32x2_t, uint8x8_t, uint8x8_t, __builtin_riscv_pm4addu_u8x8)
+__packed_binary_builtin_mixed(pm2addu_u16x4, uint32x2_t, uint16x4_t, uint16x4_t, __builtin_riscv_pm2addu_u16x4)
+__packed_binary_builtin_mixed(pmq2add_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pmq2add_i16x4)
+__packed_binary_builtin_mixed(pmqr2add_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pmqr2add_i16x4)
+__packed_binary_builtin_mixed(pm2sadd_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2sadd_i16x4)
+__packed_binary_builtin_mixed(pm2sadd_x_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2sadd_x_i16x4)
+__packed_binary_builtin_mixed(pm2sub_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2sub_i16x4)
+__packed_binary_builtin_mixed(pm2sub_x_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2sub_x_i16x4)
+__packed_binary_builtin_mixed(pm4addsu_i8x8, int32x2_t, int8x8_t, uint8x8_t, __builtin_riscv_pm4addsu_i8x8)
+__packed_binary_builtin_mixed(pm2addsu_i16x4, int32x2_t, int16x4_t, uint16x4_t, __builtin_riscv_pm2addsu_i16x4)
+__packed_binary_builtin_mixed(pm2add_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pm2add_i32x2)
+__packed_binary_builtin_mixed(pm2add_x_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pm2add_x_i32x2)
+__packed_binary_builtin_mixed(pm2addu_u32x2, uint64_t, uint32x2_t, uint32x2_t, __builtin_riscv_pm2addu_u32x2)
+__packed_binary_builtin_mixed(pmq2add_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pmq2add_i32x2)
+__packed_binary_builtin_mixed(pm2sub_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pm2sub_i32x2)
+__packed_binary_builtin_mixed(pm2sub_x_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pm2sub_x_i32x2)
+__packed_binary_builtin_mixed(pm2addsu_i32x2, int64_t, int32x2_t, uint32x2_t, __builtin_riscv_pm2addsu_i32x2)
+__packed_binary_builtin_mixed(pmqr2add_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pmqr2add_i32x2)
+__packed_binary_builtin_mixed(pm4add_i16x4, int64_t, int16x4_t, int16x4_t, __builtin_riscv_pm4add_i16x4)
+__packed_binary_builtin_mixed(pm4addu_u16x4, uint64_t, uint16x4_t, uint16x4_t, __builtin_riscv_pm4addu_u16x4)
+__packed_binary_builtin_mixed(pm4addsu_i16x4, int64_t, int16x4_t, uint16x4_t, __builtin_riscv_pm4addsu_i16x4)
+
/* Packed Absolute Difference Sum (32-bit) */
__packed_abdsum(pabdsumu_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumu_u8x4_u32)
__packed_ternary_builtin_cast(pabdsumau_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumau_u8x4_u32)
diff --git a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
index 089db9a7fa511..d76b2b0a4c7db 100644
--- a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
+++ b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
@@ -3607,6 +3607,291 @@ int64_t test_mqracc_w11_i64(int64_t rd, int32x2_t a, int32x2_t b) {
return __riscv_mqracc_w11_i64(rd, a, b);
}
+// CHECK-LABEL: test_pm4add_i8x4:
+// RV32: pm4add.b
+// RV64: pm4add.b
+int32_t test_pm4add_i8x4(int8x4_t rs1, int8x4_t rs2) {
+ return __riscv_pm4add_i8x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2add_i16x2:
+// RV32: pm2add.h
+// RV64: pm2add.h
+int32_t test_pm2add_i16x2(int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2add_i16x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2add_x_i16x2:
+// RV32: pm2add.hx
+// RV64: pm2add.hx
+int32_t test_pm2add_x_i16x2(int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2add_x_i16x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm4addu_u8x4:
+// RV32: pm4addu.b
+// RV64: pm4addu.b
+uint32_t test_pm4addu_u8x4(uint8x4_t rs1, uint8x4_t rs2) {
+ return __riscv_pm4addu_u8x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2addu_u16x2:
+// RV32: pm2addu.h
+// RV64: pm2addu.h
+uint32_t test_pm2addu_u16x2(uint16x2_t rs1, uint16x2_t rs2) {
+ return __riscv_pm2addu_u16x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmq2add_i16x2:
+// RV32: pmq2add.h
+// RV64: pmq2add.h
+int32_t test_pmq2add_i16x2(int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pmq2add_i16x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmqr2add_i16x2:
+// RV32: pmqr2add.h
+// RV64: pmqr2add.h
+int32_t test_pmqr2add_i16x2(int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pmqr2add_i16x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2sadd_i16x2:
+// RV32: pm2sadd.h
+// RV64: pm2sadd.h
+int32_t test_pm2sadd_i16x2(int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2sadd_i16x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2sadd_x_i16x2:
+// RV32: pm2sadd.hx
+// RV64: pm2sadd.hx
+int32_t test_pm2sadd_x_i16x2(int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2sadd_x_i16x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2sub_i16x2:
+// RV32: pm2sub.h
+// RV64: pm2sub.h
+int32_t test_pm2sub_i16x2(int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2sub_i16x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2sub_x_i16x2:
+// RV32: pm2sub.hx
+// RV64: pm2sub.hx
+int32_t test_pm2sub_x_i16x2(int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2sub_x_i16x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm4addsu_i8x4:
+// RV32: pm4addsu.b
+// RV64: pm4addsu.b
+int32_t test_pm4addsu_i8x4(int8x4_t rs1, uint8x4_t rs2) {
+ return __riscv_pm4addsu_i8x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2addsu_i16x2:
+// RV32: pm2addsu.h
+// RV64: pm2addsu.h
+int32_t test_pm2addsu_i16x2(int16x2_t rs1, uint16x2_t rs2) {
+ return __riscv_pm2addsu_i16x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm4add_i8x8:
+// RV32: pm4add.b
+// RV32: pm4add.b
+// RV64: pm4add.b
+int32x2_t test_pm4add_i8x8(int8x8_t rs1, int8x8_t rs2) {
+ return __riscv_pm4add_i8x8(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2add_i16x4:
+// RV32: pm2add.h
+// RV32: pm2add.h
+// RV64: pm2add.h
+int32x2_t test_pm2add_i16x4(int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pm2add_i16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2add_x_i16x4:
+// RV32: pm2add.hx
+// RV32: pm2add.hx
+// RV64: pm2add.hx
+int32x2_t test_pm2add_x_i16x4(int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pm2add_x_i16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm4addu_u8x8:
+// RV32: pm4addu.b
+// RV32: pm4addu.b
+// RV64: pm4addu.b
+uint32x2_t test_pm4addu_u8x8(uint8x8_t rs1, uint8x8_t rs2) {
+ return __riscv_pm4addu_u8x8(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2addu_u16x4:
+// RV32: pm2addu.h
+// RV32: pm2addu.h
+// RV64: pm2addu.h
+uint32x2_t test_pm2addu_u16x4(uint16x4_t rs1, uint16x4_t rs2) {
+ return __riscv_pm2addu_u16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmq2add_i16x4:
+// RV32: pmq2add.h
+// RV32: pmq2add.h
+// RV64: pmq2add.h
+int32x2_t test_pmq2add_i16x4(int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pmq2add_i16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmqr2add_i16x4:
+// RV32: pmqr2add.h
+// RV32: pmqr2add.h
+// RV64: pmqr2add.h
+int32x2_t test_pmqr2add_i16x4(int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pmqr2add_i16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2sadd_i16x4:
+// RV32: pm2sadd.h
+// RV32: pm2sadd.h
+// RV64: pm2sadd.h
+int32x2_t test_pm2sadd_i16x4(int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pm2sadd_i16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2sadd_x_i16x4:
+// RV32: pm2sadd.hx
+// RV32: pm2sadd.hx
+// RV64: pm2sadd.hx
+int32x2_t test_pm2sadd_x_i16x4(int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pm2sadd_x_i16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2sub_i16x4:
+// RV32: pm2sub.h
+// RV32: pm2sub.h
+// RV64: pm2sub.h
+int32x2_t test_pm2sub_i16x4(int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pm2sub_i16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2sub_x_i16x4:
+// RV32: pm2sub.hx
+// RV32: pm2sub.hx
+// RV64: pm2sub.hx
+int32x2_t test_pm2sub_x_i16x4(int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pm2sub_x_i16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm4addsu_i8x8:
+// RV32: pm4addsu.b
+// RV32: pm4addsu.b
+// RV64: pm4addsu.b
+int32x2_t test_pm4addsu_i8x8(int8x8_t rs1, uint8x8_t rs2) {
+ return __riscv_pm4addsu_i8x8(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2addsu_i16x4:
+// RV32: pm2addsu.h
+// RV32: pm2addsu.h
+// RV64: pm2addsu.h
+int32x2_t test_pm2addsu_i16x4(int16x4_t rs1, uint16x4_t rs2) {
+ return __riscv_pm2addsu_i16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2add_i32x2:
+// RV32: wmul
+// RV32: wmacc
+// RV64: pm2add.w
+int64_t test_pm2add_i32x2(int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_pm2add_i32x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2add_x_i32x2:
+// RV32: wmul
+// RV32: wmacc
+// RV64: pm2add.wx
+int64_t test_pm2add_x_i32x2(int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_pm2add_x_i32x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2addu_u32x2:
+// RV32: wmulu
+// RV32: wmaccu
+// RV64: pm2addu.w
+uint64_t test_pm2addu_u32x2(uint32x2_t rs1, uint32x2_t rs2) {
+ return __riscv_pm2addu_u32x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmq2add_i32x2:
+// RV32: mqwacc
+// RV32: mqwacc
+// RV64: pmq2add.w
+int64_t test_pmq2add_i32x2(int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_pmq2add_i32x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmqr2add_i32x2:
+// RV32: mqrwacc
+// RV32: mqrwacc
+// RV64: pmqr2add.w
+int64_t test_pmqr2add_i32x2(int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_pmqr2add_i32x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2sub_i32x2:
+// RV32: wmul
+// RV32: wmul
+// RV32: subd
+// RV64: pm2sub.w
+int64_t test_pm2sub_i32x2(int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_pm2sub_i32x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2sub_x_i32x2:
+// RV32: wmul
+// RV32: wmul
+// RV32: subd
+// RV64: pm2sub.wx
+int64_t test_pm2sub_x_i32x2(int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_pm2sub_x_i32x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm2addsu_i32x2:
+// RV32: wmulsu
+// RV32: wmaccsu
+// RV64: pm2addsu.w
+int64_t test_pm2addsu_i32x2(int32x2_t rs1, uint32x2_t rs2) {
+ return __riscv_pm2addsu_i32x2(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm4add_i16x4:
+// RV32: pm2wadd.h
+// RV32: pm2wadda.h
+// RV64: pm4add.h
+int64_t test_pm4add_i16x4(int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pm4add_i16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm4addu_u16x4:
+// RV32: pm2waddu.h
+// RV32: pm2waddau.h
+// RV64: pm4addu.h
+uint64_t test_pm4addu_u16x4(uint16x4_t rs1, uint16x4_t rs2) {
+ return __riscv_pm4addu_u16x4(rs1, rs2);
+}
+
+// CHECK-LABEL: test_pm4addsu_i16x4:
+// RV32: pm2waddsu.h
+// RV32: pm2waddasu.h
+// RV64: pm4addsu.h
+int64_t test_pm4addsu_i16x4(int16x4_t rs1, uint16x4_t rs2) {
+ return __riscv_pm4addsu_i16x4(rs1, rs2);
+}
+
// Packed Multiply Parts.
// CHECK-LABEL: test_pmul_b00_i16x2:
// RV32: pmul.h.b00
diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td
index 06b9af4ec644a..e4ec9e9beb5ee 100644
--- a/llvm/include/llvm/IR/IntrinsicsRISCV.td
+++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td
@@ -2136,6 +2136,25 @@ class RVPBinaryIntrinsic
def int_riscv_pmhaccsu : RVPTernaryIntrinsic;
def int_riscv_pmhraccsu : RVPTernaryIntrinsic;
+ // Packed Multiplication with Horizontal Addition.
+ class RVPHorizontalMulIntrinsic
+ : DefaultAttrsIntrinsic<[llvm_any_ty],
+ [llvm_anyvector_ty, LLVMMatchType<1>],
+ [IntrNoMem, IntrSpeculatable]>;
+ def int_riscv_pm4add : RVPHorizontalMulIntrinsic;
+ def int_riscv_pm2add : RVPHorizontalMulIntrinsic;
+ def int_riscv_pm2add_x : RVPHorizontalMulIntrinsic;
+ def int_riscv_pm4addu : RVPHorizontalMulIntrinsic;
+ def int_riscv_pm2addu : RVPHorizontalMulIntrinsic;
+ def int_riscv_pmq2add : RVPHorizontalMulIntrinsic;
+ def int_riscv_pmqr2add : RVPHorizontalMulIntrinsic;
+ def int_riscv_pm2sadd : RVPHorizontalMulIntrinsic;
+ def int_riscv_pm2sadd_x : RVPHorizontalMulIntrinsic;
+ def int_riscv_pm2sub : RVPHorizontalMulIntrinsic;
+ def int_riscv_pm2sub_x : RVPHorizontalMulIntrinsic;
+ def int_riscv_pm4addsu : RVPHorizontalMulIntrinsic;
+ def int_riscv_pm2addsu : RVPHorizontalMulIntrinsic;
+
// Packed Multiply Parts. The packed forms multiply the selected element of
// every pair, so the operands are the result with each element split in two.
class RVPPackedMulPartsIntrinsic
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 3ed01728057d6..c47c70bf821ad 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -12328,6 +12328,107 @@ static unsigned getRVPQFormatAccOpcode(Intrinsic::ID IntNo) {
}
}
+static unsigned getRVPHorizontalMulOpcode(unsigned IntNo) {
+ switch (IntNo) {
+ default:
+ llvm_unreachable("Unexpected RISC-V packed horizontal multiply intrinsic");
+ case Intrinsic::riscv_pm4add:
+ return RISCVISD::PM4ADD;
+ case Intrinsic::riscv_pm2add:
+ return RISCVISD::PM2ADD;
+ case Intrinsic::riscv_pm2add_x:
+ return RISCVISD::PM2ADD_X;
+ case Intrinsic::riscv_pm4addu:
+ return RISCVISD::PM4ADDU;
+ case Intrinsic::riscv_pm2addu:
+ return RISCVISD::PM2ADDU;
+ case Intrinsic::riscv_pmq2add:
+ return RISCVISD::PMQ2ADD;
+ case Intrinsic::riscv_pmqr2add:
+ return RISCVISD::PMQR2ADD;
+ case Intrinsic::riscv_pm2sadd:
+ return RISCVISD::PM2SADD;
+ case Intrinsic::riscv_pm2sadd_x:
+ return RISCVISD::PM2SADD_X;
+ case Intrinsic::riscv_pm2sub:
+ return RISCVISD::PM2SUB;
+ case Intrinsic::riscv_pm2sub_x:
+ return RISCVISD::PM2SUB_X;
+ case Intrinsic::riscv_pm4addsu:
+ return RISCVISD::PM4ADDSU;
+ case Intrinsic::riscv_pm2addsu:
+ return RISCVISD::PM2ADDSU;
+ }
+}
+
+static SDValue lowerRV32HorizontalMul64(unsigned IntNo, SDValue Rs1,
+ SDValue Rs2, const SDLoc &DL,
+ SelectionDAG &DAG) {
+ auto Extract = [&](SDValue V, unsigned Idx) {
+ return DAG.getExtractVectorElt(DL, MVT::i32, V, Idx);
+ };
+
+ if (Rs1.getSimpleValueType() == MVT::v4i16) {
+ auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
+ auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
+ unsigned MulOpc, AccOpc;
+ switch (IntNo) {
+ default:
+ llvm_unreachable("Unexpected RV32 horizontal multiply intrinsic");
+ case Intrinsic::riscv_pm4add:
+ MulOpc = RISCVISD::PM2WADD;
+ AccOpc = RISCVISD::PM2WADDA;
+ break;
+ case Intrinsic::riscv_pm4addu:
+ MulOpc = RISCVISD::PM2WADDU;
+ AccOpc = RISCVISD::PM2WADDAU;
+ break;
+ case Intrinsic::riscv_pm4addsu:
+ MulOpc = RISCVISD::PM2WADDSU;
+ AccOpc = RISCVISD::PM2WADDASU;
+ break;
+ }
+ SDValue Acc = DAG.getNode(MulOpc, DL, MVT::v2i32, Rs1Lo, Rs2Lo);
+ Acc = DAG.getNode(AccOpc, DL, MVT::v2i32, Acc, Rs1Hi, Rs2Hi);
+ return DAG.getBitcast(MVT::i64, Acc);
+ }
+
+ assert(Rs1.getSimpleValueType() == MVT::v2i32 &&
+ "Unexpected RV32 horizontal multiply source type");
+ SDValue Rs1Lo = Extract(Rs1, 0);
+ SDValue Rs1Hi = Extract(Rs1, 1);
+ SDValue Rs2Lo = Extract(Rs2, 0);
+ SDValue Rs2Hi = Extract(Rs2, 1);
+
+ if (IntNo == Intrinsic::riscv_pmq2add || IntNo == Intrinsic::riscv_pmqr2add) {
+ unsigned AccOpc = IntNo == Intrinsic::riscv_pmq2add ? RISCVISD::MQWACC
+ : RISCVISD::MQRWACC;
+ SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32);
+ SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
+ SDValue Acc = DAG.getNode(AccOpc, DL, VTs, {Zero, Zero, Rs1Lo, Rs2Lo});
+ Acc = DAG.getNode(AccOpc, DL, VTs, {Acc, Acc.getValue(1), Rs1Hi, Rs2Hi});
+ return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Acc, Acc.getValue(1));
+ }
+
+ unsigned MulOpc = ISD::SMUL_LOHI;
+ if (IntNo == Intrinsic::riscv_pm2addu)
+ MulOpc = ISD::UMUL_LOHI;
+ else if (IntNo == Intrinsic::riscv_pm2addsu)
+ MulOpc = RISCVISD::WMULSU;
+ bool IsSub =
+ IntNo == Intrinsic::riscv_pm2sub || IntNo == Intrinsic::riscv_pm2sub_x;
+ if (IntNo == Intrinsic::riscv_pm2add_x || IntNo == Intrinsic::riscv_pm2sub_x)
+ std::swap(Rs2Lo, Rs2Hi);
+
+ SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32);
+ SDValue LoMul = DAG.getNode(MulOpc, DL, VTs, Rs1Lo, Rs2Lo);
+ SDValue HiMul = DAG.getNode(MulOpc, DL, VTs, Rs1Hi, Rs2Hi);
+ unsigned Opc = IsSub ? RISCVISD::SUBD : RISCVISD::ADDD;
+ SDValue Res = DAG.getNode(Opc, DL, VTs, LoMul, LoMul.getValue(1), HiMul,
+ HiMul.getValue(1));
+ return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Res, Res.getValue(1));
+}
+
/// Return the packed multiply-halves node for a multiply-parts intrinsic. The
/// scalar spelling maps to the same node; its product is the first element.
static unsigned getRVPMulHalvesOpcode(unsigned IntNo) {
@@ -12884,6 +12985,36 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
return DAG.getNode(MulOpc, DL, VT, Rd, Rs1, Rs2);
}
+ case Intrinsic::riscv_pm4add:
+ case Intrinsic::riscv_pm2add:
+ case Intrinsic::riscv_pm2add_x:
+ case Intrinsic::riscv_pm4addu:
+ case Intrinsic::riscv_pm2addu:
+ case Intrinsic::riscv_pmq2add:
+ case Intrinsic::riscv_pmqr2add:
+ case Intrinsic::riscv_pm2sadd:
+ case Intrinsic::riscv_pm2sadd_x:
+ case Intrinsic::riscv_pm2sub:
+ case Intrinsic::riscv_pm2sub_x:
+ case Intrinsic::riscv_pm4addsu:
+ case Intrinsic::riscv_pm2addsu: {
+ EVT VT = Op.getValueType();
+ unsigned Opc = getRVPHorizontalMulOpcode(IntNo);
+ SDValue Rs1 = Op.getOperand(1);
+ SDValue Rs2 = Op.getOperand(2);
+
+ // RV32 applies the 32-bit instruction independently to both halves of a
+ // 64-bit packed input.
+ if (!Subtarget.is64Bit() && VT == MVT::v2i32) {
+ auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
+ auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
+ SDValue Lo = DAG.getNode(Opc, DL, MVT::i32, Rs1Lo, Rs2Lo);
+ SDValue Hi = DAG.getNode(Opc, DL, MVT::i32, Rs1Hi, Rs2Hi);
+ return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Lo, Hi);
+ }
+
+ return DAG.getNode(Opc, DL, VT, Rs1, Rs2);
+ }
case Intrinsic::riscv_pmerge: {
EVT VT = Op.getValueType();
auto buildMerge = [&](SDValue Rs1, SDValue Rs2, SDValue Mask,
@@ -16975,6 +17106,43 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
Results.push_back(DAG.getExtractSubvector(DL, VT, Res, 0));
return;
}
+ case Intrinsic::riscv_pm4add:
+ case Intrinsic::riscv_pm2add:
+ case Intrinsic::riscv_pm2add_x:
+ case Intrinsic::riscv_pm4addu:
+ case Intrinsic::riscv_pm2addu:
+ case Intrinsic::riscv_pmq2add:
+ case Intrinsic::riscv_pmqr2add:
+ case Intrinsic::riscv_pm2sadd:
+ case Intrinsic::riscv_pm2sadd_x:
+ case Intrinsic::riscv_pm2sub:
+ case Intrinsic::riscv_pm2sub_x:
+ case Intrinsic::riscv_pm4addsu:
+ case Intrinsic::riscv_pm2addsu: {
+ MVT VT = N->getSimpleValueType(0);
+ unsigned Opc = getRVPHorizontalMulOpcode(IntNo);
+ SDValue Rs1 = N->getOperand(1);
+ SDValue Rs2 = N->getOperand(2);
+
+ if (!Subtarget.is64Bit() && VT == MVT::i64) {
+ SDValue Res = lowerRV32HorizontalMul64(IntNo, Rs1, Rs2, DL, DAG);
+ Results.push_back(Res);
+ return;
+ }
+
+ assert(Subtarget.is64Bit() && VT == MVT::i32 &&
+ "Unexpected horizontal multiply legalization");
+ MVT SrcVT = Rs1.getSimpleValueType();
+ MVT WideSrcVT = SrcVT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16;
+ Rs1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideSrcVT, Rs1,
+ DAG.getUNDEF(SrcVT));
+ Rs2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideSrcVT, Rs2,
+ DAG.getUNDEF(SrcVT));
+ SDValue Wide = DAG.getNode(Opc, DL, MVT::v2i32, Rs1, Rs2);
+ Results.push_back(DAG.getExtractVectorElt(DL, MVT::i32, Wide, 0));
+ return;
+ }
+
case Intrinsic::riscv_pmul_00:
case Intrinsic::riscv_pmul_01:
case Intrinsic::riscv_pmul_11:
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
index fce4418109e0a..c7372ef3549b6 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
@@ -1883,6 +1883,28 @@ def riscv_pmulsu_halves_00
def riscv_pmulsu_halves_11
: RVSDNode<"PMULSU_HALVES_11", SDT_RISCVPackedWideningMulByHalves>;
+// Packed multiplication with widening horizontal addition.
+def SDT_RISCVPackedWideningHorizontalAdd
+ : SDTypeProfile<1, 2, [SDTCisVT<0, v2i32>, SDTCisVT<1, v2i16>,
+ SDTCisSameAs<1, 2>]>;
+def riscv_pm2wadd
+ : RVSDNode<"PM2WADD", SDT_RISCVPackedWideningHorizontalAdd>;
+def riscv_pm2waddu
+ : RVSDNode<"PM2WADDU", SDT_RISCVPackedWideningHorizontalAdd>;
+def riscv_pm2waddsu
+ : RVSDNode<"PM2WADDSU", SDT_RISCVPackedWideningHorizontalAdd>;
+
+// Packed multiplication with widening horizontal addition and accumulate.
+def SDT_RISCVPackedWideningHorizontalAddAcc
+ : SDTypeProfile<1, 3, [SDTCisVT<0, v2i32>, SDTCisSameAs<0, 1>,
+ SDTCisVT<2, v2i16>, SDTCisSameAs<2, 3>]>;
+def riscv_pm2wadda
+ : RVSDNode<"PM2WADDA", SDT_RISCVPackedWideningHorizontalAddAcc>;
+def riscv_pm2waddau
+ : RVSDNode<"PM2WADDAU", SDT_RISCVPackedWideningHorizontalAddAcc>;
+def riscv_pm2waddasu
+ : RVSDNode<"PM2WADDASU", SDT_RISCVPackedWideningHorizontalAddAcc>;
+
def SDT_RISCVPM2Halfword
: SDTypeProfile<1, 2, [SDTCisVT<0, v2i32>,
SDTCisVT<1, v4i16>,
@@ -1949,6 +1971,23 @@ def riscv_mhraccu : RVSDNode<"MHRACCU", SDT_RISCVSameTernary>;
def riscv_mhaccsu : RVSDNode<"MHACCSU", SDT_RISCVSameTernary>;
def riscv_mhraccsu : RVSDNode<"MHRACCSU", SDT_RISCVSameTernary>;
+// Multiply packed elements and horizontally combine each adjacent group.
+def SDT_RISCVHorizontalMul
+ : SDTypeProfile<1, 2, [SDTCisVec<1>, SDTCisSameAs<1, 2>]>;
+def riscv_pm4add : RVSDNode<"PM4ADD", SDT_RISCVHorizontalMul>;
+def riscv_pm2add : RVSDNode<"PM2ADD", SDT_RISCVHorizontalMul>;
+def riscv_pm2add_x : RVSDNode<"PM2ADD_X", SDT_RISCVHorizontalMul>;
+def riscv_pm4addu : RVSDNode<"PM4ADDU", SDT_RISCVHorizontalMul>;
+def riscv_pm2addu : RVSDNode<"PM2ADDU", SDT_RISCVHorizontalMul>;
+def riscv_pmq2add : RVSDNode<"PMQ2ADD", SDT_RISCVHorizontalMul>;
+def riscv_pmqr2add : RVSDNode<"PMQR2ADD", SDT_RISCVHorizontalMul>;
+def riscv_pm2sadd : RVSDNode<"PM2SADD", SDT_RISCVHorizontalMul>;
+def riscv_pm2sadd_x : RVSDNode<"PM2SADD_X", SDT_RISCVHorizontalMul>;
+def riscv_pm2sub : RVSDNode<"PM2SUB", SDT_RISCVHorizontalMul>;
+def riscv_pm2sub_x : RVSDNode<"PM2SUB_X", SDT_RISCVHorizontalMul>;
+def riscv_pm4addsu : RVSDNode<"PM4ADDSU", SDT_RISCVHorizontalMul>;
+def riscv_pm2addsu : RVSDNode<"PM2ADDSU", SDT_RISCVHorizontalMul>;
+
// "Q-format" multiplication
def riscv_mulq : RVSDNode<"MULQ", SDTIntBinOp, [SDNPCommutative]>;
def riscv_mulqr : RVSDNode<"MULQR", SDTIntBinOp, [SDNPCommutative]>;
@@ -2077,6 +2116,11 @@ def RISCVBuildPairGPRVec : RVSDNode<"BuildPairGPRVec",
SDT_RISCVBuildPairGPRVec>;
def RISCVSplitGPRVec : RVSDNode<"SplitGPRVec", SDT_RISCVSplitGPRVec>;
+class PatHorizontalMul<SDPatternOperator OpNode, RVInst Inst,
+ ValueType ResultVT, ValueType SrcVT>
+ : Pat<(ResultVT (OpNode (SrcVT GPR:$rs1), (SrcVT GPR:$rs2))),
+ (Inst GPR:$rs1, GPR:$rs2)>;
+
// (rs1 << 1) + rs2
class PatPSh1Add<RVInst Inst, ValueType vt>
: Pat<(vt (add (riscv_pshl (vt GPR:$rs1), (XLenVT 1)), (vt GPR:$rs2))),
@@ -2390,6 +2434,21 @@ let append Predicates = [IsRV32] in {
def : PatGprGpr<riscv_mulq, MULQ, i32>;
def : PatGprGpr<riscv_mulqr, MULQR, i32>;
+ // Packed multiplication with horizontal addition patterns.
+ def : PatHorizontalMul<riscv_pm4add, PM4ADD_B, i32, v4i8>;
+ def : PatHorizontalMul<riscv_pm2add, PM2ADD_H, i32, v2i16>;
+ def : PatHorizontalMul<riscv_pm2add_x, PM2ADD_HX, i32, v2i16>;
+ def : PatHorizontalMul<riscv_pm4addu, PM4ADDU_B, i32, v4i8>;
+ def : PatHorizontalMul<riscv_pm2addu, PM2ADDU_H, i32, v2i16>;
+ def : PatHorizontalMul<riscv_pmq2add, PMQ2ADD_H, i32, v2i16>;
+ def : PatHorizontalMul<riscv_pmqr2add, PMQR2ADD_H, i32, v2i16>;
+ def : PatHorizontalMul<riscv_pm2sadd, PM2SADD_H, i32, v2i16>;
+ def : PatHorizontalMul<riscv_pm2sadd_x, PM2SADD_HX, i32, v2i16>;
+ def : PatHorizontalMul<riscv_pm2sub, PM2SUB_H, i32, v2i16>;
+ def : PatHorizontalMul<riscv_pm2sub_x, PM2SUB_HX, i32, v2i16>;
+ def : PatHorizontalMul<riscv_pm4addsu, PM4ADDSU_B, i32, v4i8>;
+ def : PatHorizontalMul<riscv_pm2addsu, PM2ADDSU_H, i32, v2i16>;
+
// Halfword multiply patterns where one operand is a sext.h or zext.h and
// the other is a sext.h or zext.h or is known to be sign/zero-extended. We
// prefer plain mul when both operands are known to be sign/zero-extended.
@@ -2473,6 +2532,26 @@ let append Predicates = [IsRV32] in {
def : Pat<(v2i32 (riscv_pwmulsu (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
(PWMULSU_H GPR:$rs1, GPR:$rs2)>;
+ // Packed multiplication with widening horizontal addition patterns.
+ def : Pat<(v2i32 (riscv_pm2wadd (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
+ (PM2WADD_H GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(v2i32 (riscv_pm2waddu (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
+ (PM2WADDU_H GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(v2i32 (riscv_pm2waddsu (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
+ (PM2WADDSU_H GPR:$rs1, GPR:$rs2)>;
+
+ // Packed multiplication with widening horizontal addition and accumulate
+ // patterns.
+ def : Pat<(v2i32 (riscv_pm2wadda (v2i32 GPRPair:$rd),
+ (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
+ (PM2WADDA_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(v2i32 (riscv_pm2waddau (v2i32 GPRPair:$rd),
+ (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
+ (PM2WADDAU_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(v2i32 (riscv_pm2waddasu (v2i32 GPRPair:$rd),
+ (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
+ (PM2WADDASU_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>;
+
// 8/16-bit bitreverse patterns
// With Zbkb, brev8 reverses the bits within each byte directly; otherwise
// reverse all bits then swap the bytes back.
@@ -3026,6 +3105,33 @@ let append Predicates = [IsRV64] in {
def : PatGprGpr<riscv_mulq, PMULQ_W, v2i32>;
def : PatGprGpr<riscv_mulqr, PMULQR_W, v2i32>;
+ // Packed multiplication with horizontal addition patterns.
+ def : PatHorizontalMul<riscv_pm4add, PM4ADD_B, v2i32, v8i8>;
+ def : PatHorizontalMul<riscv_pm2add, PM2ADD_H, v2i32, v4i16>;
+ def : PatHorizontalMul<riscv_pm2add_x, PM2ADD_HX, v2i32, v4i16>;
+ def : PatHorizontalMul<riscv_pm4addu, PM4ADDU_B, v2i32, v8i8>;
+ def : PatHorizontalMul<riscv_pm2addu, PM2ADDU_H, v2i32, v4i16>;
+ def : PatHorizontalMul<riscv_pmq2add, PMQ2ADD_H, v2i32, v4i16>;
+ def : PatHorizontalMul<riscv_pmqr2add, PMQR2ADD_H, v2i32, v4i16>;
+ def : PatHorizontalMul<riscv_pm2sadd, PM2SADD_H, v2i32, v4i16>;
+ def : PatHorizontalMul<riscv_pm2sadd_x, PM2SADD_HX, v2i32, v4i16>;
+ def : PatHorizontalMul<riscv_pm2sub, PM2SUB_H, v2i32, v4i16>;
+ def : PatHorizontalMul<riscv_pm2sub_x, PM2SUB_HX, v2i32, v4i16>;
+ def : PatHorizontalMul<riscv_pm4addsu, PM4ADDSU_B, v2i32, v8i8>;
+ def : PatHorizontalMul<riscv_pm2addsu, PM2ADDSU_H, v2i32, v4i16>;
+
+ def : PatHorizontalMul<riscv_pm2add, PM2ADD_W, i64, v2i32>;
+ def : PatHorizontalMul<riscv_pm2add_x, PM2ADD_WX, i64, v2i32>;
+ def : PatHorizontalMul<riscv_pm2addu, PM2ADDU_W, i64, v2i32>;
+ def : PatHorizontalMul<riscv_pmq2add, PMQ2ADD_W, i64, v2i32>;
+ def : PatHorizontalMul<riscv_pmqr2add, PMQR2ADD_W, i64, v2i32>;
+ def : PatHorizontalMul<riscv_pm2sub, PM2SUB_W, i64, v2i32>;
+ def : PatHorizontalMul<riscv_pm2sub_x, PM2SUB_WX, i64, v2i32>;
+ def : PatHorizontalMul<riscv_pm2addsu, PM2ADDSU_W, i64, v2i32>;
+ def : PatHorizontalMul<riscv_pm4add, PM4ADD_H, i64, v4i16>;
+ def : PatHorizontalMul<riscv_pm4addu, PM4ADDU_H, i64, v4i16>;
+ def : PatHorizontalMul<riscv_pm4addsu, PM4ADDSU_H, i64, v4i16>;
+
// 8/16/32-bit multiply low patterns
// FIXME custom lower
def : Pat<(v8i8 (mul GPR:$rs1, GPR:$rs2)),
diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-32.ll b/llvm/test/CodeGen/RISCV/rvp-simd-32.ll
index fbdf685fb55bc..c11d9716ec053 100644
--- a/llvm/test/CodeGen/RISCV/rvp-simd-32.ll
+++ b/llvm/test/CodeGen/RISCV/rvp-simd-32.ll
@@ -3588,3 +3588,120 @@ define i32 @test_mulsu_h11_i32(<2 x i16> %a, <2 x i16> %b) {
%r = call i32 @llvm.riscv.mulsu.11.i32.v2i16(<2 x i16> %a, <2 x i16> %b)
ret i32 %r
}
+
+define i32 @test_pm4add_v4i8_i32(<4 x i8> %a, <4 x i8> %b) {
+; CHECK-LABEL: test_pm4add_v4i8_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm4add.b a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm4add.i32.v4i8(<4 x i8> %a, <4 x i8> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2add_v2i16_i32(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2add_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2add.h a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2add.i32.v2i16(<2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2add_x_v2i16_i32(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2add_x_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2add.hx a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2add.x.i32.v2i16(<2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm4addu_v4i8_i32(<4 x i8> %a, <4 x i8> %b) {
+; CHECK-LABEL: test_pm4addu_v4i8_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm4addu.b a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm4addu.i32.v4i8(<4 x i8> %a, <4 x i8> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2addu_v2i16_i32(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2addu_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2addu.h a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2addu.i32.v2i16(<2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pmq2add_v2i16_i32(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pmq2add_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pmq2add.h a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pmq2add.i32.v2i16(<2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pmqr2add_v2i16_i32(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pmqr2add_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pmqr2add.h a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pmqr2add.i32.v2i16(<2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2sadd_v2i16_i32(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2sadd_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2sadd.h a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2sadd.i32.v2i16(<2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2sadd_x_v2i16_i32(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2sadd_x_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2sadd.hx a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2sadd.x.i32.v2i16(<2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2sub_v2i16_i32(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2sub_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2sub.h a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2sub.i32.v2i16(<2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2sub_x_v2i16_i32(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2sub_x_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2sub.hx a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2sub.x.i32.v2i16(<2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm4addsu_v4i8_i32(<4 x i8> %a, <4 x i8> %b) {
+; CHECK-LABEL: test_pm4addsu_v4i8_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm4addsu.b a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm4addsu.i32.v4i8(<4 x i8> %a, <4 x i8> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2addsu_v2i16_i32(<2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2addsu_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2addsu.h a0, a0, a1
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2addsu.i32.v2i16(<2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
index b3790659ff9c1..fd9cf65c2ef53 100644
--- a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
+++ b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
@@ -7491,3 +7491,378 @@ define i64 @test_mulsu_w11_i64(<2 x i32> %a, <2 x i32> %b) {
%r = call i64 @llvm.riscv.mulsu.11.i64.v2i32(<2 x i32> %a, <2 x i32> %b)
ret i64 %r
}
+
+define <2 x i32> @test_pm4add_v8i8_v2i32(<8 x i8> %a, <8 x i8> %b) {
+; RV32-LABEL: test_pm4add_v8i8_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm4add.b a1, a1, a3
+; RV32-NEXT: pm4add.b a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm4add_v8i8_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm4add.b a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm4add.v2i32.v8i8(<8 x i8> %a, <8 x i8> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm2add_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2add_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2add.h a1, a1, a3
+; RV32-NEXT: pm2add.h a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2add_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2add.h a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2add.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm2add_x_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2add_x_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2add.hx a1, a1, a3
+; RV32-NEXT: pm2add.hx a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2add_x_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2add.hx a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2add.x.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm4addu_v8i8_v2i32(<8 x i8> %a, <8 x i8> %b) {
+; RV32-LABEL: test_pm4addu_v8i8_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm4addu.b a1, a1, a3
+; RV32-NEXT: pm4addu.b a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm4addu_v8i8_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm4addu.b a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm4addu.v2i32.v8i8(<8 x i8> %a, <8 x i8> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm2addu_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2addu_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2addu.h a1, a1, a3
+; RV32-NEXT: pm2addu.h a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2addu_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2addu.h a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2addu.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pmq2add_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pmq2add_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pmq2add.h a1, a1, a3
+; RV32-NEXT: pmq2add.h a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pmq2add_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pmq2add.h a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pmq2add.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pmqr2add_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pmqr2add_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pmqr2add.h a1, a1, a3
+; RV32-NEXT: pmqr2add.h a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pmqr2add_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pmqr2add.h a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pmqr2add.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm2sadd_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2sadd_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2sadd.h a1, a1, a3
+; RV32-NEXT: pm2sadd.h a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2sadd_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2sadd.h a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2sadd.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm2sadd_x_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2sadd_x_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2sadd.hx a1, a1, a3
+; RV32-NEXT: pm2sadd.hx a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2sadd_x_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2sadd.hx a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2sadd.x.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm2sub_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2sub_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2sub.h a1, a1, a3
+; RV32-NEXT: pm2sub.h a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2sub_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2sub.h a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2sub.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm2sub_x_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2sub_x_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2sub.hx a1, a1, a3
+; RV32-NEXT: pm2sub.hx a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2sub_x_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2sub.hx a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2sub.x.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm4addsu_v8i8_v2i32(<8 x i8> %a, <8 x i8> %b) {
+; RV32-LABEL: test_pm4addsu_v8i8_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm4addsu.b a1, a1, a3
+; RV32-NEXT: pm4addsu.b a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm4addsu_v8i8_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm4addsu.b a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm4addsu.v2i32.v8i8(<8 x i8> %a, <8 x i8> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm2addsu_v4i16_v2i32(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2addsu_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2addsu.h a1, a1, a3
+; RV32-NEXT: pm2addsu.h a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2addsu_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2addsu.h a0, a0, a1
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2addsu.v2i32.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define i64 @test_pm2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pm2add_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mv a4, a0
+; RV32-NEXT: wmul a0, a1, a3
+; RV32-NEXT: wmacc a0, a4, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2add_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2add.w a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2add.i64.v2i32(<2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm2add_x_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pm2add_x_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mv a4, a0
+; RV32-NEXT: wmul a0, a1, a2
+; RV32-NEXT: wmacc a0, a4, a3
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2add_x_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2add.wx a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2add.x.i64.v2i32(<2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm2addu_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pm2addu_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mv a4, a0
+; RV32-NEXT: wmulu a0, a1, a3
+; RV32-NEXT: wmaccu a0, a4, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2addu_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2addu.w a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2addu.i64.v2i32(<2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pmq2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pmq2add_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: li a4, 0
+; RV32-NEXT: li a5, 0
+; RV32-NEXT: mqwacc a4, a0, a2
+; RV32-NEXT: mqwacc a4, a1, a3
+; RV32-NEXT: mvd a0, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pmq2add_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pmq2add.w a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pmq2add.i64.v2i32(<2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pmqr2add_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pmqr2add_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: li a4, 0
+; RV32-NEXT: li a5, 0
+; RV32-NEXT: mqrwacc a4, a0, a2
+; RV32-NEXT: mqrwacc a4, a1, a3
+; RV32-NEXT: mvd a0, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pmqr2add_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pmqr2add.w a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pmqr2add.i64.v2i32(<2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm2sub_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pm2sub_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: wmul a4, a1, a3
+; RV32-NEXT: wmul a0, a0, a2
+; RV32-NEXT: subd a0, a0, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2sub_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2sub.w a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2sub.i64.v2i32(<2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm2sub_x_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pm2sub_x_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: wmul a4, a1, a2
+; RV32-NEXT: wmul a0, a0, a3
+; RV32-NEXT: subd a0, a0, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2sub_x_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2sub.wx a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2sub.x.i64.v2i32(<2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm2addsu_v2i32_i64(<2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pm2addsu_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mv a4, a0
+; RV32-NEXT: wmulsu a0, a1, a3
+; RV32-NEXT: wmaccsu a0, a4, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2addsu_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2addsu.w a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2addsu.i64.v2i32(<2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm4add_v4i16_i64(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm4add_v4i16_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mv a4, a1
+; RV32-NEXT: pm2wadd.h a0, a0, a2
+; RV32-NEXT: pm2wadda.h a0, a4, a3
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm4add_v4i16_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm4add.h a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm4add.i64.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm4addu_v4i16_i64(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm4addu_v4i16_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mv a4, a1
+; RV32-NEXT: pm2waddu.h a0, a0, a2
+; RV32-NEXT: pm2waddau.h a0, a4, a3
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm4addu_v4i16_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm4addu.h a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm4addu.i64.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm4addsu_v4i16_i64(<4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm4addsu_v4i16_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mv a4, a1
+; RV32-NEXT: pm2waddsu.h a0, a0, a2
+; RV32-NEXT: pm2waddasu.h a0, a4, a3
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm4addsu_v4i16_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm4addsu.h a0, a0, a1
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm4addsu.i64.v4i16(<4 x i16> %a, <4 x i16> %b)
+ ret i64 %r
+}
More information about the llvm-commits
mailing list