[clang] [llvm] [RISCV][P-ext] Support Packed Multiplication with Horizontal Addition and Accumulate intrinsics (PR #236303)
via cfe-commits
cfe-commits at lists.llvm.org
Mon Nov 2 22:47:10 PST 2026
https://github.com/TelGome updated https://github.com/llvm/llvm-project/pull/236303
>From f03b8cd1a076a236489b2f5adcfa04b6b1335096 Mon Sep 17 00:00:00 2001
From: Dongyan Chen <chendongyan at isrc.iscas.ac.cn>
Date: Wed, 28 Oct 2026 02:14:25 +0000
Subject: [PATCH 1/5] [RISCV][P-ext] Support Packed Multiplication with
Horizontal Addition and Accumulate intrinsics
Add the remaining 30 intrinsics of the "Packed Multiplication with
Horizontal Addition and Accumulate".
---
clang/include/clang/Basic/BuiltinsRISCV.td | 34 ++
clang/lib/CodeGen/TargetBuiltins/RISCV.cpp | 92 ++++
clang/lib/Headers/riscv_packed_simd.h | 34 ++
clang/test/CodeGen/RISCV/rvp-intrinsics.c | 466 ++++++++++++++++++
.../riscv_packed_simd.c | 251 +++++++++-
llvm/include/llvm/IR/IntrinsicsRISCV.td | 13 +
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 208 ++++++++
llvm/lib/Target/RISCV/RISCVInstrInfoP.td | 63 +++
llvm/test/CodeGen/RISCV/rvp-simd-64.ll | 389 +++++++++++++++
9 files changed, 1540 insertions(+), 10 deletions(-)
diff --git a/clang/include/clang/Basic/BuiltinsRISCV.td b/clang/include/clang/Basic/BuiltinsRISCV.td
index 704547a6830048..5f10c8dfd49210 100644
--- a/clang/include/clang/Basic/BuiltinsRISCV.td
+++ b/clang/include/clang/Basic/BuiltinsRISCV.td
@@ -408,6 +408,40 @@ def pm4add_i16x4 : RISCVBuiltin<"int64_t(_Vector<4, short>, _Vector<4, short>)">
def pm4addu_u16x4 : RISCVBuiltin<"uint64_t(_Vector<4, unsigned short>, _Vector<4, unsigned short>)">;
def pm4addsu_i16x4 : RISCVBuiltin<"int64_t(_Vector<4, short>, _Vector<4, unsigned short>)">;
+// Packed Multiplication with Horizontal Addition and Accumulate (32-bit)
+def pm4adda_i8x4 : RISCVBuiltin<"int(int, _Vector<4, signed char>, _Vector<4, signed char>)">;
+def pm4addau_u8x4 : RISCVBuiltin<"unsigned int(unsigned int, _Vector<4, unsigned char>, _Vector<4, unsigned char>)">;
+def pm4addasu_i8x4 : RISCVBuiltin<"int(int, _Vector<4, signed char>, _Vector<4, unsigned char>)">;
+def pm2adda_i16x2 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short>)">;
+def pm2adda_x_i16x2 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short>)">;
+def pm2addau_u16x2 : RISCVBuiltin<"unsigned int(unsigned int, _Vector<2, unsigned short>, _Vector<2, unsigned short>)">;
+def pmq2adda_i16x2 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short>)">;
+def pmqr2adda_i16x2 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short>)">;
+def pm2suba_i16x2 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short>)">;
+def pm2suba_x_i16x2 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short>)">;
+def pm2addasu_i16x2 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, unsigned short>)">;
+
+// Packed Multiplication with Horizontal Addition and Accumulate (64-bit)
+def pm2adda_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">;
+def pm2adda_x_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">;
+def pm2addau_u16x4 : RISCVBuiltin<"_Vector<2, unsigned int>(_Vector<2, unsigned int>, _Vector<4, unsigned short>, _Vector<4, unsigned short>)">;
+def pmq2adda_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">;
+def pmqr2adda_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">;
+def pm2suba_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">;
+def pm2suba_x_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">;
+def pm2addasu_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, unsigned short>)">;
+def pm2adda_i32x2 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">;
+def pm2adda_x_i32x2 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">;
+def pm2addau_u32x2 : RISCVBuiltin<"uint64_t(uint64_t, _Vector<2, unsigned int>, _Vector<2, unsigned int>)">;
+def pmq2adda_i32x2 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">;
+def pmqr2adda_i32x2 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">;
+def pm2suba_i32x2 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">;
+def pm2suba_x_i32x2 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">;
+def pm2addasu_i32x2 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, unsigned int>)">;
+def pm4adda_i16x4 : RISCVBuiltin<"int64_t(int64_t, _Vector<4, short>, _Vector<4, short>)">;
+def pm4addau_u16x4 : RISCVBuiltin<"uint64_t(uint64_t, _Vector<4, unsigned short>, _Vector<4, unsigned short>)">;
+def pm4addasu_i16x4 : RISCVBuiltin<"int64_t(int64_t, _Vector<4, short>, _Vector<4, unsigned short>)">;
+
// Packed Widening Multiply Accumulate
def pwmacc_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<2, short>, _Vector<2, short>)">;
def pwmaccu_u32x2 : RISCVBuiltin<"_Vector<2, unsigned int>(_Vector<2, unsigned int>, _Vector<2, unsigned short>, _Vector<2, unsigned short>)">;
diff --git a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
index 0d6bc6c66c1a01..e61f8950b51bac 100644
--- a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
@@ -1968,6 +1968,98 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID,
break;
}
+ // Packed Multiplication with Horizontal Addition and Accumulate.
+ case RISCV::BI__builtin_riscv_pm2adda_i16x2:
+ case RISCV::BI__builtin_riscv_pm2adda_i16x4:
+ case RISCV::BI__builtin_riscv_pm2adda_i32x2:
+ case RISCV::BI__builtin_riscv_pm2adda_x_i16x2:
+ case RISCV::BI__builtin_riscv_pm2adda_x_i16x4:
+ case RISCV::BI__builtin_riscv_pm2adda_x_i32x2:
+ case RISCV::BI__builtin_riscv_pm2addau_u16x2:
+ case RISCV::BI__builtin_riscv_pm2addau_u16x4:
+ case RISCV::BI__builtin_riscv_pm2addau_u32x2:
+ case RISCV::BI__builtin_riscv_pmq2adda_i16x2:
+ case RISCV::BI__builtin_riscv_pmq2adda_i16x4:
+ case RISCV::BI__builtin_riscv_pmq2adda_i32x2:
+ case RISCV::BI__builtin_riscv_pmqr2adda_i16x2:
+ case RISCV::BI__builtin_riscv_pmqr2adda_i16x4:
+ case RISCV::BI__builtin_riscv_pmqr2adda_i32x2:
+ case RISCV::BI__builtin_riscv_pm2suba_i16x2:
+ case RISCV::BI__builtin_riscv_pm2suba_i16x4:
+ case RISCV::BI__builtin_riscv_pm2suba_i32x2:
+ case RISCV::BI__builtin_riscv_pm2suba_x_i16x2:
+ case RISCV::BI__builtin_riscv_pm2suba_x_i16x4:
+ case RISCV::BI__builtin_riscv_pm2suba_x_i32x2:
+ case RISCV::BI__builtin_riscv_pm2addasu_i16x2:
+ case RISCV::BI__builtin_riscv_pm2addasu_i16x4:
+ case RISCV::BI__builtin_riscv_pm2addasu_i32x2:
+ case RISCV::BI__builtin_riscv_pm4adda_i8x4:
+ case RISCV::BI__builtin_riscv_pm4adda_i16x4:
+ case RISCV::BI__builtin_riscv_pm4addau_u8x4:
+ case RISCV::BI__builtin_riscv_pm4addau_u16x4:
+ case RISCV::BI__builtin_riscv_pm4addasu_i8x4:
+ case RISCV::BI__builtin_riscv_pm4addasu_i16x4: {
+ switch (BuiltinID) {
+ default:
+ llvm_unreachable("unexpected builtin ID");
+ case RISCV::BI__builtin_riscv_pm2adda_i16x2:
+ case RISCV::BI__builtin_riscv_pm2adda_i16x4:
+ case RISCV::BI__builtin_riscv_pm2adda_i32x2:
+ ID = Intrinsic::riscv_pm2adda;
+ break;
+ case RISCV::BI__builtin_riscv_pm2adda_x_i16x2:
+ case RISCV::BI__builtin_riscv_pm2adda_x_i16x4:
+ case RISCV::BI__builtin_riscv_pm2adda_x_i32x2:
+ ID = Intrinsic::riscv_pm2adda_x;
+ break;
+ case RISCV::BI__builtin_riscv_pm2addau_u16x2:
+ case RISCV::BI__builtin_riscv_pm2addau_u16x4:
+ case RISCV::BI__builtin_riscv_pm2addau_u32x2:
+ ID = Intrinsic::riscv_pm2addau;
+ break;
+ case RISCV::BI__builtin_riscv_pmq2adda_i16x2:
+ case RISCV::BI__builtin_riscv_pmq2adda_i16x4:
+ case RISCV::BI__builtin_riscv_pmq2adda_i32x2:
+ ID = Intrinsic::riscv_pmq2adda;
+ break;
+ case RISCV::BI__builtin_riscv_pmqr2adda_i16x2:
+ case RISCV::BI__builtin_riscv_pmqr2adda_i16x4:
+ case RISCV::BI__builtin_riscv_pmqr2adda_i32x2:
+ ID = Intrinsic::riscv_pmqr2adda;
+ break;
+ case RISCV::BI__builtin_riscv_pm2suba_i16x2:
+ case RISCV::BI__builtin_riscv_pm2suba_i16x4:
+ case RISCV::BI__builtin_riscv_pm2suba_i32x2:
+ ID = Intrinsic::riscv_pm2suba;
+ break;
+ case RISCV::BI__builtin_riscv_pm2suba_x_i16x2:
+ case RISCV::BI__builtin_riscv_pm2suba_x_i16x4:
+ case RISCV::BI__builtin_riscv_pm2suba_x_i32x2:
+ ID = Intrinsic::riscv_pm2suba_x;
+ break;
+ case RISCV::BI__builtin_riscv_pm2addasu_i16x2:
+ case RISCV::BI__builtin_riscv_pm2addasu_i16x4:
+ case RISCV::BI__builtin_riscv_pm2addasu_i32x2:
+ ID = Intrinsic::riscv_pm2addasu;
+ break;
+ case RISCV::BI__builtin_riscv_pm4adda_i8x4:
+ case RISCV::BI__builtin_riscv_pm4adda_i16x4:
+ ID = Intrinsic::riscv_pm4adda;
+ break;
+ case RISCV::BI__builtin_riscv_pm4addau_u8x4:
+ case RISCV::BI__builtin_riscv_pm4addau_u16x4:
+ ID = Intrinsic::riscv_pm4addau;
+ break;
+ case RISCV::BI__builtin_riscv_pm4addasu_i8x4:
+ case RISCV::BI__builtin_riscv_pm4addasu_i16x4:
+ ID = Intrinsic::riscv_pm4addasu;
+ break;
+ }
+
+ IntrinsicTypes = {ResultType, Ops[1]->getType()};
+ break;
+ }
+
// Packed Reduction Sum
case RISCV::BI__builtin_riscv_predsum_i8x4_i32:
case RISCV::BI__builtin_riscv_predsum_i16x2_i32:
diff --git a/clang/lib/Headers/riscv_packed_simd.h b/clang/lib/Headers/riscv_packed_simd.h
index 89f678a63e94e6..4f3af7d39dc520 100644
--- a/clang/lib/Headers/riscv_packed_simd.h
+++ b/clang/lib/Headers/riscv_packed_simd.h
@@ -1220,6 +1220,20 @@ __packed_ternary_builtin_mixed(pwmacc_i32x2, int32x2_t, int16x2_t, int16x2_t, __
__packed_ternary_builtin_mixed(pwmaccu_u32x2, uint32x2_t, uint16x2_t, uint16x2_t, __builtin_riscv_pwmaccu_u32x2)
__packed_ternary_builtin_mixed(pwmaccsu_i32x2, int32x2_t, int16x2_t, uint16x2_t, __builtin_riscv_pwmaccsu_i32x2)
+/* Packed Multiplication with Horizontal Addition and Accumulate (32-bit) */
+__packed_ternary_builtin_mixed(pm4adda_i8x4, int32_t, int8x4_t, int8x4_t, __builtin_riscv_pm4adda_i8x4)
+__packed_ternary_builtin_mixed(pm4addau_u8x4, uint32_t, uint8x4_t, uint8x4_t, __builtin_riscv_pm4addau_u8x4)
+__packed_ternary_builtin_mixed(pm4addasu_i8x4, int32_t, int8x4_t, uint8x4_t, __builtin_riscv_pm4addasu_i8x4)
+__packed_ternary_builtin_mixed(pm2adda_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2adda_i16x2)
+__packed_ternary_builtin_mixed(pm2adda_x_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2adda_x_i16x2)
+__packed_ternary_builtin_mixed(pm2addau_u16x2, uint32_t, uint16x2_t, uint16x2_t, __builtin_riscv_pm2addau_u16x2)
+__packed_ternary_builtin_mixed(pmq2adda_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pmq2adda_i16x2)
+__packed_ternary_builtin_mixed(pmqr2adda_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pmqr2adda_i16x2)
+__packed_ternary_builtin_mixed(pm2suba_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2suba_i16x2)
+__packed_ternary_builtin_mixed(pm2suba_x_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2suba_x_i16x2)
+__packed_ternary_builtin_mixed(pm2addasu_i16x2, int32_t, int16x2_t, uint16x2_t, __builtin_riscv_pm2addasu_i16x2)
+
+
/* Packed Multiplication with Horizontal Addition and Accumulate (64-bit) */
static __inline__ int32x2_t __DEFAULT_FN_ATTRS
__riscv_pm4adda_i8x8(int32x2_t __rd, int8x8_t __rs1, int8x8_t __rs2) {
@@ -1238,6 +1252,26 @@ __riscv_pm4addasu_i8x8(int32x2_t __rd, int8x8_t __rs1, uint8x8_t __rs2) {
(uint32x2_t)__riscv_pm4addsu_i8x8(__rs1, __rs2));
}
+__packed_ternary_builtin_mixed(pm2adda_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2adda_i16x4)
+__packed_ternary_builtin_mixed(pm2adda_x_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2adda_x_i16x4)
+__packed_ternary_builtin_mixed(pm2addau_u16x4, uint32x2_t, uint16x4_t, uint16x4_t, __builtin_riscv_pm2addau_u16x4)
+__packed_ternary_builtin_mixed(pmq2adda_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pmq2adda_i16x4)
+__packed_ternary_builtin_mixed(pmqr2adda_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pmqr2adda_i16x4)
+__packed_ternary_builtin_mixed(pm2suba_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2suba_i16x4)
+__packed_ternary_builtin_mixed(pm2suba_x_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2suba_x_i16x4)
+__packed_ternary_builtin_mixed(pm2addasu_i16x4, int32x2_t, int16x4_t, uint16x4_t, __builtin_riscv_pm2addasu_i16x4)
+__packed_ternary_builtin_mixed(pm2adda_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pm2adda_i32x2)
+__packed_ternary_builtin_mixed(pm2adda_x_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pm2adda_x_i32x2)
+__packed_ternary_builtin_mixed(pm2addau_u32x2, uint64_t, uint32x2_t, uint32x2_t, __builtin_riscv_pm2addau_u32x2)
+__packed_ternary_builtin_mixed(pmq2adda_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pmq2adda_i32x2)
+__packed_ternary_builtin_mixed(pmqr2adda_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pmqr2adda_i32x2)
+__packed_ternary_builtin_mixed(pm2suba_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pm2suba_i32x2)
+__packed_ternary_builtin_mixed(pm2suba_x_i32x2, int64_t, int32x2_t, int32x2_t, __builtin_riscv_pm2suba_x_i32x2)
+__packed_ternary_builtin_mixed(pm2addasu_i32x2, int64_t, int32x2_t, uint32x2_t, __builtin_riscv_pm2addasu_i32x2)
+__packed_ternary_builtin_mixed(pm4adda_i16x4, int64_t, int16x4_t, int16x4_t, __builtin_riscv_pm4adda_i16x4)
+__packed_ternary_builtin_mixed(pm4addau_u16x4, uint64_t, uint16x4_t, uint16x4_t, __builtin_riscv_pm4addau_u16x4)
+__packed_ternary_builtin_mixed(pm4addasu_i16x4, int64_t, int16x4_t, uint16x4_t, __builtin_riscv_pm4addasu_i16x4)
+
/* Packed Multiplication with Widening Horizontal Addition */
__packed_binary_builtin_mixed(pm2wadd_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wadd_i64)
__packed_binary_builtin_mixed(pm2wadd_x_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wadd_x_i64)
diff --git a/clang/test/CodeGen/RISCV/rvp-intrinsics.c b/clang/test/CodeGen/RISCV/rvp-intrinsics.c
index 0fa253c1cba893..598db5ca5863c2 100644
--- a/clang/test/CodeGen/RISCV/rvp-intrinsics.c
+++ b/clang/test/CodeGen/RISCV/rvp-intrinsics.c
@@ -13026,3 +13026,469 @@ int64_t test_maccsu_w00_i64(int64_t rd, int32x2_t a, uint32x2_t b) {
int64_t test_maccsu_w11_i64(int64_t rd, int32x2_t a, uint32x2_t b) {
return __riscv_maccsu_w11_i64(rd, a, b);
}
+
+/* Packed Multiplication with Horizontal Addition and Accumulate */
+
+// RV32-LABEL: define dso_local i32 @test_pm4adda_i8x4(
+// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <4 x i8>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <4 x i8>
+// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pm4adda.i32.v4i8(i32 [[RD]], <4 x i8> [[TMP0]], <4 x i8> [[TMP1]])
+// RV32-NEXT: ret i32 [[TMP2]]
+//
+// RV64-LABEL: define dso_local signext i32 @test_pm4adda_i8x4(
+// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <4 x i8>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <4 x i8>
+// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pm4adda.i32.v4i8(i32 [[RD]], <4 x i8> [[TMP0]], <4 x i8> [[TMP1]])
+// RV64-NEXT: ret i32 [[TMP2]]
+//
+int32_t test_pm4adda_i8x4(int32_t rd, int8x4_t rs1, int8x4_t rs2) {
+ return __riscv_pm4adda_i8x4(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i32 @test_pm4addau_u8x4(
+// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <4 x i8>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <4 x i8>
+// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pm4addau.i32.v4i8(i32 [[RD]], <4 x i8> [[TMP0]], <4 x i8> [[TMP1]])
+// RV32-NEXT: ret i32 [[TMP2]]
+//
+// RV64-LABEL: define dso_local signext i32 @test_pm4addau_u8x4(
+// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <4 x i8>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <4 x i8>
+// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pm4addau.i32.v4i8(i32 [[RD]], <4 x i8> [[TMP0]], <4 x i8> [[TMP1]])
+// RV64-NEXT: ret i32 [[TMP2]]
+//
+uint32_t test_pm4addau_u8x4(uint32_t rd, uint8x4_t rs1, uint8x4_t rs2) {
+ return __riscv_pm4addau_u8x4(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i32 @test_pm4addasu_i8x4(
+// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <4 x i8>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <4 x i8>
+// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pm4addasu.i32.v4i8(i32 [[RD]], <4 x i8> [[TMP0]], <4 x i8> [[TMP1]])
+// RV32-NEXT: ret i32 [[TMP2]]
+//
+// RV64-LABEL: define dso_local signext i32 @test_pm4addasu_i8x4(
+// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <4 x i8>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <4 x i8>
+// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pm4addasu.i32.v4i8(i32 [[RD]], <4 x i8> [[TMP0]], <4 x i8> [[TMP1]])
+// RV64-NEXT: ret i32 [[TMP2]]
+//
+int32_t test_pm4addasu_i8x4(int32_t rd, int8x4_t rs1, uint8x4_t rs2) {
+ return __riscv_pm4addasu_i8x4(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i32 @test_pm2adda_i16x2(
+// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pm2adda.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV32-NEXT: ret i32 [[TMP2]]
+//
+// RV64-LABEL: define dso_local signext i32 @test_pm2adda_i16x2(
+// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pm2adda.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV64-NEXT: ret i32 [[TMP2]]
+//
+int32_t test_pm2adda_i16x2(int32_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2adda_i16x2(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i32 @test_pm2adda_x_i16x2(
+// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pm2adda.x.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV32-NEXT: ret i32 [[TMP2]]
+//
+// RV64-LABEL: define dso_local signext i32 @test_pm2adda_x_i16x2(
+// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pm2adda.x.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV64-NEXT: ret i32 [[TMP2]]
+//
+int32_t test_pm2adda_x_i16x2(int32_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2adda_x_i16x2(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i32 @test_pm2addau_u16x2(
+// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pm2addau.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV32-NEXT: ret i32 [[TMP2]]
+//
+// RV64-LABEL: define dso_local signext i32 @test_pm2addau_u16x2(
+// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pm2addau.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV64-NEXT: ret i32 [[TMP2]]
+//
+uint32_t test_pm2addau_u16x2(uint32_t rd, uint16x2_t rs1, uint16x2_t rs2) {
+ return __riscv_pm2addau_u16x2(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i32 @test_pmq2adda_i16x2(
+// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pmq2adda.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV32-NEXT: ret i32 [[TMP2]]
+//
+// RV64-LABEL: define dso_local signext i32 @test_pmq2adda_i16x2(
+// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pmq2adda.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV64-NEXT: ret i32 [[TMP2]]
+//
+int32_t test_pmq2adda_i16x2(int32_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pmq2adda_i16x2(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i32 @test_pmqr2adda_i16x2(
+// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pmqr2adda.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV32-NEXT: ret i32 [[TMP2]]
+//
+// RV64-LABEL: define dso_local signext i32 @test_pmqr2adda_i16x2(
+// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pmqr2adda.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV64-NEXT: ret i32 [[TMP2]]
+//
+int32_t test_pmqr2adda_i16x2(int32_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pmqr2adda_i16x2(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i32 @test_pm2suba_i16x2(
+// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pm2suba.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV32-NEXT: ret i32 [[TMP2]]
+//
+// RV64-LABEL: define dso_local signext i32 @test_pm2suba_i16x2(
+// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pm2suba.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV64-NEXT: ret i32 [[TMP2]]
+//
+int32_t test_pm2suba_i16x2(int32_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2suba_i16x2(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i32 @test_pm2suba_x_i16x2(
+// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pm2suba.x.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV32-NEXT: ret i32 [[TMP2]]
+//
+// RV64-LABEL: define dso_local signext i32 @test_pm2suba_x_i16x2(
+// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pm2suba.x.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV64-NEXT: ret i32 [[TMP2]]
+//
+int32_t test_pm2suba_x_i16x2(int32_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pm2suba_x_i16x2(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i32 @test_pm2addasu_i16x2(
+// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pm2addasu.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV32-NEXT: ret i32 [[TMP2]]
+//
+// RV64-LABEL: define dso_local signext i32 @test_pm2addasu_i16x2(
+// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.pm2addasu.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV64-NEXT: ret i32 [[TMP2]]
+//
+int32_t test_pm2addasu_i16x2(int32_t rd, int16x2_t rs1, uint16x2_t rs2) {
+ return __riscv_pm2addasu_i16x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pm2adda_i16x4(
+// CHECK-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pm2adda.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+// CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
+// CHECK-NEXT: ret i64 [[TMP4]]
+//
+int32x2_t test_pm2adda_i16x4(int32x2_t rd, int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pm2adda_i16x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pm2adda_x_i16x4(
+// CHECK-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pm2adda.x.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+// CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
+// CHECK-NEXT: ret i64 [[TMP4]]
+//
+int32x2_t test_pm2adda_x_i16x4(int32x2_t rd, int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pm2adda_x_i16x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pm2addau_u16x4(
+// CHECK-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pm2addau.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+// CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
+// CHECK-NEXT: ret i64 [[TMP4]]
+//
+uint32x2_t test_pm2addau_u16x4(uint32x2_t rd, uint16x4_t rs1, uint16x4_t rs2) {
+ return __riscv_pm2addau_u16x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pmq2adda_i16x4(
+// CHECK-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmq2adda.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+// CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
+// CHECK-NEXT: ret i64 [[TMP4]]
+//
+int32x2_t test_pmq2adda_i16x4(int32x2_t rd, int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pmq2adda_i16x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pmqr2adda_i16x4(
+// CHECK-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqr2adda.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+// CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
+// CHECK-NEXT: ret i64 [[TMP4]]
+//
+int32x2_t test_pmqr2adda_i16x4(int32x2_t rd, int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pmqr2adda_i16x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pm2suba_i16x4(
+// CHECK-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pm2suba.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+// CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
+// CHECK-NEXT: ret i64 [[TMP4]]
+//
+int32x2_t test_pm2suba_i16x4(int32x2_t rd, int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pm2suba_i16x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pm2suba_x_i16x4(
+// CHECK-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pm2suba.x.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+// CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
+// CHECK-NEXT: ret i64 [[TMP4]]
+//
+int32x2_t test_pm2suba_x_i16x4(int32x2_t rd, int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pm2suba_x_i16x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pm2addasu_i16x4(
+// CHECK-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pm2addasu.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+// CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
+// CHECK-NEXT: ret i64 [[TMP4]]
+//
+int32x2_t test_pm2addasu_i16x4(int32x2_t rd, int16x4_t rs1, uint16x4_t rs2) {
+ return __riscv_pm2addasu_i16x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pm2adda_i32x2(
+// CHECK-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2adda.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]])
+// CHECK-NEXT: ret i64 [[TMP2]]
+//
+int64_t test_pm2adda_i32x2(int64_t rd, int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_pm2adda_i32x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pm2adda_x_i32x2(
+// CHECK-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2adda.x.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]])
+// CHECK-NEXT: ret i64 [[TMP2]]
+//
+int64_t test_pm2adda_x_i32x2(int64_t rd, int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_pm2adda_x_i32x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pm2addau_u32x2(
+// CHECK-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2addau.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]])
+// CHECK-NEXT: ret i64 [[TMP2]]
+//
+uint64_t test_pm2addau_u32x2(uint64_t rd, uint32x2_t rs1, uint32x2_t rs2) {
+ return __riscv_pm2addau_u32x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pmq2adda_i32x2(
+// CHECK-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pmq2adda.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]])
+// CHECK-NEXT: ret i64 [[TMP2]]
+//
+int64_t test_pmq2adda_i32x2(int64_t rd, int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_pmq2adda_i32x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pmqr2adda_i32x2(
+// CHECK-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pmqr2adda.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]])
+// CHECK-NEXT: ret i64 [[TMP2]]
+//
+int64_t test_pmqr2adda_i32x2(int64_t rd, int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_pmqr2adda_i32x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pm2suba_i32x2(
+// CHECK-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2suba.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]])
+// CHECK-NEXT: ret i64 [[TMP2]]
+//
+int64_t test_pm2suba_i32x2(int64_t rd, int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_pm2suba_i32x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pm2suba_x_i32x2(
+// CHECK-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2suba.x.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]])
+// CHECK-NEXT: ret i64 [[TMP2]]
+//
+int64_t test_pm2suba_x_i32x2(int64_t rd, int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_pm2suba_x_i32x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pm2addasu_i32x2(
+// CHECK-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32>
+// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2addasu.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]])
+// CHECK-NEXT: ret i64 [[TMP2]]
+//
+int64_t test_pm2addasu_i32x2(int64_t rd, int32x2_t rs1, uint32x2_t rs2) {
+ return __riscv_pm2addasu_i32x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pm4adda_i16x4(
+// CHECK-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm4adda.i64.v4i16(i64 [[RD]], <4 x i16> [[TMP0]], <4 x i16> [[TMP1]])
+// CHECK-NEXT: ret i64 [[TMP2]]
+//
+int64_t test_pm4adda_i16x4(int64_t rd, int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pm4adda_i16x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pm4addau_u16x4(
+// CHECK-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm4addau.i64.v4i16(i64 [[RD]], <4 x i16> [[TMP0]], <4 x i16> [[TMP1]])
+// CHECK-NEXT: ret i64 [[TMP2]]
+//
+uint64_t test_pm4addau_u16x4(uint64_t rd, uint16x4_t rs1, uint16x4_t rs2) {
+ return __riscv_pm4addau_u16x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: define dso_local i64 @test_pm4addasu_i16x4(
+// CHECK-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm4addasu.i64.v4i16(i64 [[RD]], <4 x i16> [[TMP0]], <4 x i16> [[TMP1]])
+// CHECK-NEXT: ret i64 [[TMP2]]
+//
+int64_t test_pm4addasu_i16x4(int64_t rd, int16x4_t rs1, uint16x4_t rs2) {
+ return __riscv_pm4addasu_i16x4(rd, rs1, rs2);
+}
diff --git a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
index 8ce714923c25d8..d19560c1fff471 100644
--- a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
+++ b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
@@ -4642,31 +4642,262 @@ int32x2_t test_pwmaccsu_i32x2(int32x2_t rd, int16x2_t rs1, uint16x2_t rs2) {
return __riscv_pwmaccsu_i32x2(rd, rs1, rs2);
}
-// Packed Byte Multiplication with Horizontal Addition and Accumulate.
+// Packed Multiplication with Horizontal Addition and Accumulate.
+// CHECK-LABEL: test_pm4adda_i8x4:
+// RV32: pm4adda.b
+// RV64: pm4adda.b
+int32_t test_pm4adda_i8x4(int32_t rd, int8x4_t a, int8x4_t b) {
+ return __riscv_pm4adda_i8x4(rd, a, b);
+}
+
+// CHECK-LABEL: test_pm2adda_i16x2:
+// RV32: pm2adda.h
+// RV64: pm2adda.h
+int32_t test_pm2adda_i16x2(int32_t rd, int16x2_t a, int16x2_t b) {
+ return __riscv_pm2adda_i16x2(rd, a, b);
+}
+
+// CHECK-LABEL: test_pm2adda_x_i16x2:
+// RV32: pm2adda.hx
+// RV64: pm2adda.hx
+int32_t test_pm2adda_x_i16x2(int32_t rd, int16x2_t a, int16x2_t b) {
+ return __riscv_pm2adda_x_i16x2(rd, a, b);
+}
+
+// CHECK-LABEL: test_pm4addau_u8x4:
+// RV32: pm4addau.b
+// RV64: pm4addau.b
+uint32_t test_pm4addau_u8x4(uint32_t rd, uint8x4_t a, uint8x4_t b) {
+ return __riscv_pm4addau_u8x4(rd, a, b);
+}
+
+// CHECK-LABEL: test_pm2addau_u16x2:
+// RV32: pm2addau.h
+// RV64: pm2addau.h
+uint32_t test_pm2addau_u16x2(uint32_t rd, uint16x2_t a, uint16x2_t b) {
+ return __riscv_pm2addau_u16x2(rd, a, b);
+}
+
+// CHECK-LABEL: test_pmq2adda_i16x2:
+// RV32: pmq2adda.h
+// RV64: pmq2adda.h
+int32_t test_pmq2adda_i16x2(int32_t rd, int16x2_t a, int16x2_t b) {
+ return __riscv_pmq2adda_i16x2(rd, a, b);
+}
+
+// CHECK-LABEL: test_pmqr2adda_i16x2:
+// RV32: pmqr2adda.h
+// RV64: pmqr2adda.h
+int32_t test_pmqr2adda_i16x2(int32_t rd, int16x2_t a, int16x2_t b) {
+ return __riscv_pmqr2adda_i16x2(rd, a, b);
+}
+
+// CHECK-LABEL: test_pm2suba_i16x2:
+// RV32: pm2suba.h
+// RV64: pm2suba.h
+int32_t test_pm2suba_i16x2(int32_t rd, int16x2_t a, int16x2_t b) {
+ return __riscv_pm2suba_i16x2(rd, a, b);
+}
+
+// CHECK-LABEL: test_pm2suba_x_i16x2:
+// RV32: pm2suba.hx
+// RV64: pm2suba.hx
+int32_t test_pm2suba_x_i16x2(int32_t rd, int16x2_t a, int16x2_t b) {
+ return __riscv_pm2suba_x_i16x2(rd, a, b);
+}
+
+// CHECK-LABEL: test_pm4addasu_i8x4:
+// RV32: pm4addasu.b
+// RV64: pm4addasu.b
+int32_t test_pm4addasu_i8x4(int32_t rd, int8x4_t a, uint8x4_t b) {
+ return __riscv_pm4addasu_i8x4(rd, a, b);
+}
+
+// CHECK-LABEL: test_pm2addasu_i16x2:
+// RV32: pm2addasu.h
+// RV64: pm2addasu.h
+int32_t test_pm2addasu_i16x2(int32_t rd, int16x2_t a, uint16x2_t b) {
+ return __riscv_pm2addasu_i16x2(rd, a, b);
+}
+
// CHECK-LABEL: test_pm4adda_i8x8:
-// RV32: pm4adda.b{{[[:space:]]}}
-// RV32: pm4adda.b{{[[:space:]]}}
-// RV64: pm4adda.b{{[[:space:]]}}
+// RV32: pm4adda.b
+// RV32: pm4adda.b
+// RV64: pm4adda.b
int32x2_t test_pm4adda_i8x8(int32x2_t rd, int8x8_t a, int8x8_t b) {
return __riscv_pm4adda_i8x8(rd, a, b);
}
+// CHECK-LABEL: test_pm2adda_i16x4:
+// RV32: pm2adda.h
+// RV32: pm2adda.h
+// RV64: pm2adda.h
+int32x2_t test_pm2adda_i16x4(int32x2_t rd, int16x4_t a, int16x4_t b) {
+ return __riscv_pm2adda_i16x4(rd, a, b);
+}
+
+// CHECK-LABEL: test_pm2adda_x_i16x4:
+// RV32: pm2adda.hx
+// RV32: pm2adda.hx
+// RV64: pm2adda.hx
+int32x2_t test_pm2adda_x_i16x4(int32x2_t rd, int16x4_t a, int16x4_t b) {
+ return __riscv_pm2adda_x_i16x4(rd, a, b);
+}
+
// CHECK-LABEL: test_pm4addau_u8x8:
-// RV32: pm4addau.b{{[[:space:]]}}
-// RV32: pm4addau.b{{[[:space:]]}}
-// RV64: pm4addau.b{{[[:space:]]}}
+// RV32: pm4addau.b
+// RV32: pm4addau.b
+// RV64: pm4addau.b
uint32x2_t test_pm4addau_u8x8(uint32x2_t rd, uint8x8_t a, uint8x8_t b) {
return __riscv_pm4addau_u8x8(rd, a, b);
}
+// CHECK-LABEL: test_pm2addau_u16x4:
+// RV32: pm2addau.h
+// RV32: pm2addau.h
+// RV64: pm2addau.h
+uint32x2_t test_pm2addau_u16x4(uint32x2_t rd, uint16x4_t a, uint16x4_t b) {
+ return __riscv_pm2addau_u16x4(rd, a, b);
+}
+
+// CHECK-LABEL: test_pmq2adda_i16x4:
+// RV32: pmq2adda.h
+// RV32: pmq2adda.h
+// RV64: pmq2adda.h
+int32x2_t test_pmq2adda_i16x4(int32x2_t rd, int16x4_t a, int16x4_t b) {
+ return __riscv_pmq2adda_i16x4(rd, a, b);
+}
+
+// CHECK-LABEL: test_pmqr2adda_i16x4:
+// RV32: pmqr2adda.h
+// RV32: pmqr2adda.h
+// RV64: pmqr2adda.h
+int32x2_t test_pmqr2adda_i16x4(int32x2_t rd, int16x4_t a, int16x4_t b) {
+ return __riscv_pmqr2adda_i16x4(rd, a, b);
+}
+
+// CHECK-LABEL: test_pm2suba_i16x4:
+// RV32: pm2suba.h
+// RV32: pm2suba.h
+// RV64: pm2suba.h
+int32x2_t test_pm2suba_i16x4(int32x2_t rd, int16x4_t a, int16x4_t b) {
+ return __riscv_pm2suba_i16x4(rd, a, b);
+}
+
+// CHECK-LABEL: test_pm2suba_x_i16x4:
+// RV32: pm2suba.hx
+// RV32: pm2suba.hx
+// RV64: pm2suba.hx
+int32x2_t test_pm2suba_x_i16x4(int32x2_t rd, int16x4_t a, int16x4_t b) {
+ return __riscv_pm2suba_x_i16x4(rd, a, b);
+}
+
// CHECK-LABEL: test_pm4addasu_i8x8:
-// RV32: pm4addasu.b{{[[:space:]]}}
-// RV32: pm4addasu.b{{[[:space:]]}}
-// RV64: pm4addasu.b{{[[:space:]]}}
+// RV32: pm4addasu.b
+// RV32: pm4addasu.b
+// RV64: pm4addasu.b
int32x2_t test_pm4addasu_i8x8(int32x2_t rd, int8x8_t a, uint8x8_t b) {
return __riscv_pm4addasu_i8x8(rd, a, b);
}
+// CHECK-LABEL: test_pm2addasu_i16x4:
+// RV32: pm2addasu.h
+// RV32: pm2addasu.h
+// RV64: pm2addasu.h
+int32x2_t test_pm2addasu_i16x4(int32x2_t rd, int16x4_t a, uint16x4_t b) {
+ return __riscv_pm2addasu_i16x4(rd, a, b);
+}
+
+// CHECK-LABEL: test_pm2adda_i32x2:
+// RV32: wmacc
+// RV32: wmacc
+// RV64: pm2adda.w
+int64_t test_pm2adda_i32x2(int64_t rd, int32x2_t a, int32x2_t b) {
+ return __riscv_pm2adda_i32x2(rd, a, b);
+}
+
+// CHECK-LABEL: test_pm2adda_x_i32x2:
+// RV32: wmacc
+// RV32: wmacc
+// RV64: pm2adda.wx
+int64_t test_pm2adda_x_i32x2(int64_t rd, int32x2_t a, int32x2_t b) {
+ return __riscv_pm2adda_x_i32x2(rd, a, b);
+}
+
+// CHECK-LABEL: test_pm2addau_u32x2:
+// RV32: wmaccu
+// RV32: wmaccu
+// RV64: pm2addau.w
+uint64_t test_pm2addau_u32x2(uint64_t rd, uint32x2_t a, uint32x2_t b) {
+ return __riscv_pm2addau_u32x2(rd, a, b);
+}
+
+// CHECK-LABEL: test_pmq2adda_i32x2:
+// RV32: mqwacc
+// RV32: mqwacc
+// RV64: pmq2adda.w
+int64_t test_pmq2adda_i32x2(int64_t rd, int32x2_t a, int32x2_t b) {
+ return __riscv_pmq2adda_i32x2(rd, a, b);
+}
+
+// CHECK-LABEL: test_pm2suba_i32x2:
+// RV32: wmul
+// RV32: wmacc
+// RV32: subd
+// RV64: pm2suba.w
+int64_t test_pm2suba_i32x2(int64_t rd, int32x2_t a, int32x2_t b) {
+ return __riscv_pm2suba_i32x2(rd, a, b);
+}
+
+// CHECK-LABEL: test_pm2suba_x_i32x2:
+// RV32: wmul
+// RV32: wmacc
+// RV32: subd
+// RV64: pm2suba.wx
+int64_t test_pm2suba_x_i32x2(int64_t rd, int32x2_t a, int32x2_t b) {
+ return __riscv_pm2suba_x_i32x2(rd, a, b);
+}
+
+// CHECK-LABEL: test_pm2addasu_i32x2:
+// RV32: wmaccsu
+// RV32: wmaccsu
+// RV64: pm2addasu.w
+int64_t test_pm2addasu_i32x2(int64_t rd, int32x2_t a, uint32x2_t b) {
+ return __riscv_pm2addasu_i32x2(rd, a, b);
+}
+
+// CHECK-LABEL: test_pmqr2adda_i32x2:
+// RV32: mqrwacc
+// RV32: mqrwacc
+// RV64: pmqr2adda.w
+int64_t test_pmqr2adda_i32x2(int64_t rd, int32x2_t a, int32x2_t b) {
+ return __riscv_pmqr2adda_i32x2(rd, a, b);
+}
+
+// CHECK-LABEL: test_pm4adda_i16x4:
+// RV32: pm2wadda.h
+// RV32: pm2wadda.h
+// RV64: pm4adda.h
+int64_t test_pm4adda_i16x4(int64_t rd, int16x4_t a, int16x4_t b) {
+ return __riscv_pm4adda_i16x4(rd, a, b);
+}
+
+// CHECK-LABEL: test_pm4addau_u16x4:
+// RV32: pm2waddau.h
+// RV32: pm2waddau.h
+// RV64: pm4addau.h
+uint64_t test_pm4addau_u16x4(uint64_t rd, uint16x4_t a, uint16x4_t b) {
+ return __riscv_pm4addau_u16x4(rd, a, b);
+}
+
+// CHECK-LABEL: test_pm4addasu_i16x4:
+// RV32: pm2waddasu.h
+// RV32: pm2waddasu.h
+// RV64: pm4addasu.h
+int64_t test_pm4addasu_i16x4(int64_t rd, int16x4_t a, uint16x4_t b) {
+ return __riscv_pm4addasu_i16x4(rd, a, b);
+}
+
// Packed Multiplication with Widening Horizontal Addition.
// CHECK-LABEL: test_pm2wadd_i64:
// RV32: pm2wadd.h
diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td
index 397f0f4efa9b67..d23af8c42340b2 100644
--- a/llvm/include/llvm/IR/IntrinsicsRISCV.td
+++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td
@@ -2348,6 +2348,19 @@ class RVPBinaryIntrinsic
def int_riscv_pm2wsuba_x : RVPHorizontalMulAccIntrinsic;
def int_riscv_pm2waddasu : RVPHorizontalMulAccIntrinsic;
+ // Packed Multiplication with Horizontal Addition and Accumulate.
+ def int_riscv_pm2adda : RVPHorizontalMulAccIntrinsic;
+ def int_riscv_pm2adda_x : RVPHorizontalMulAccIntrinsic;
+ def int_riscv_pm2addau : RVPHorizontalMulAccIntrinsic;
+ def int_riscv_pmq2adda : RVPHorizontalMulAccIntrinsic;
+ def int_riscv_pmqr2adda : RVPHorizontalMulAccIntrinsic;
+ def int_riscv_pm2suba : RVPHorizontalMulAccIntrinsic;
+ def int_riscv_pm2suba_x : RVPHorizontalMulAccIntrinsic;
+ def int_riscv_pm2addasu : RVPHorizontalMulAccIntrinsic;
+ def int_riscv_pm4adda : RVPHorizontalMulAccIntrinsic;
+ def int_riscv_pm4addau : RVPHorizontalMulAccIntrinsic;
+ def int_riscv_pm4addasu : RVPHorizontalMulAccIntrinsic;
+
// Packed Multiply Parts. The packed forms multiply the selected element of
// every pair, so the operands are the result with each element split in two.
class RVPPackedMulPartsIntrinsic
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index c21df65ea71d1b..329411a956deff 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -12686,6 +12686,30 @@ static unsigned getRVPHorizontalMulAccOpcode(unsigned IntNo) {
}
}
+static unsigned getRVPMulAddAccOpcode(unsigned IntNo) {
+ switch (IntNo) {
+ default:
+ llvm_unreachable("Unexpected RISC-V packed multiplication with horizontal "
+ "addition/subtraction accumulate intrinsic");
+ case Intrinsic::riscv_pm2adda:
+ return RISCVISD::PM2ADDA;
+ case Intrinsic::riscv_pm2adda_x:
+ return RISCVISD::PM2ADDA_X;
+ case Intrinsic::riscv_pm2addau:
+ return RISCVISD::PM2ADDAU;
+ case Intrinsic::riscv_pmq2adda:
+ return RISCVISD::PMQ2ADDA;
+ case Intrinsic::riscv_pmqr2adda:
+ return RISCVISD::PMQR2ADDA;
+ case Intrinsic::riscv_pm2suba:
+ return RISCVISD::PM2SUBA;
+ case Intrinsic::riscv_pm2suba_x:
+ return RISCVISD::PM2SUBA_X;
+ case Intrinsic::riscv_pm2addasu:
+ return RISCVISD::PM2ADDASU;
+ }
+}
+
static SDValue lowerRV32HorizontalMul64(unsigned IntNo, SDValue Rs1,
SDValue Rs2, const SDLoc &DL,
SelectionDAG &DAG) {
@@ -13843,6 +13867,59 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
: RISCVISD::PM4ADDA;
return DAG.getNode(AccOpc, DL, MVT::i64, Rd, A, B);
}
+ case Intrinsic::riscv_pm2adda:
+ case Intrinsic::riscv_pm2adda_x:
+ case Intrinsic::riscv_pm2addau:
+ case Intrinsic::riscv_pmq2adda:
+ case Intrinsic::riscv_pmqr2adda:
+ case Intrinsic::riscv_pm2suba:
+ case Intrinsic::riscv_pm2suba_x:
+ case Intrinsic::riscv_pm2addasu: {
+
+ MVT VT = Op.getSimpleValueType();
+ SDValue Rd = Op.getOperand(1);
+ SDValue Rs1 = Op.getOperand(2);
+ SDValue Rs2 = Op.getOperand(3);
+ unsigned Opc = getRVPMulAddAccOpcode(IntNo);
+
+ if (VT == MVT::i32)
+ return DAG.getNode(Opc, DL, MVT::i32, Rd, Rs1, Rs2);
+
+ if (VT == MVT::v2i32) {
+ if (Subtarget.is64Bit())
+ return DAG.getNode(Opc, DL, MVT::v2i32, Rd, Rs1, Rs2);
+
+ auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
+ auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
+ SDValue RdLo = DAG.getExtractVectorElt(DL, MVT::i32, Rd, 0);
+ SDValue RdHi = DAG.getExtractVectorElt(DL, MVT::i32, Rd, 1);
+ SDValue Lo = DAG.getNode(Opc, DL, MVT::i32, RdLo, Rs1Lo, Rs2Lo);
+ SDValue Hi = DAG.getNode(Opc, DL, MVT::i32, RdHi, Rs1Hi, Rs2Hi);
+ return DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2i32, Lo, Hi);
+ }
+
+ assert(Subtarget.is64Bit() && VT == MVT::i64 &&
+ "Unexpected packed multiply-horizontal-add-accumulate result");
+ return DAG.getNode(Opc, DL, MVT::i64, Rd, Rs1, Rs2);
+ }
+ case Intrinsic::riscv_pm4adda:
+ case Intrinsic::riscv_pm4addau:
+ case Intrinsic::riscv_pm4addasu: {
+
+ MVT VT = Op.getSimpleValueType();
+ SDValue Rd = Op.getOperand(1);
+ SDValue Rs1 = Op.getOperand(2);
+ SDValue Rs2 = Op.getOperand(3);
+ unsigned Opc = IntNo == Intrinsic::riscv_pm4addau ? RISCVISD::PM4ADDAU
+ : IntNo == Intrinsic::riscv_pm4addasu ? RISCVISD::PM4ADDASU
+ : RISCVISD::PM4ADDA;
+
+ if (VT == MVT::i32)
+ return DAG.getNode(Opc, DL, MVT::i32, Rd, Rs1, Rs2);
+ assert(Subtarget.is64Bit() && VT == MVT::i64 &&
+ "Unexpected packed multiply-horizontal-add-accumulate result");
+ return DAG.getNode(Opc, DL, MVT::i64, Rd, Rs1, Rs2);
+ }
case Intrinsic::riscv_pmerge: {
EVT VT = Op.getValueType();
auto buildMerge = [&](SDValue Rs1, SDValue Rs2, SDValue Mask,
@@ -18083,6 +18160,137 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
return;
}
+ case Intrinsic::riscv_pm2adda:
+ case Intrinsic::riscv_pm2adda_x:
+ case Intrinsic::riscv_pm2addau:
+ case Intrinsic::riscv_pmq2adda:
+ case Intrinsic::riscv_pmqr2adda:
+ case Intrinsic::riscv_pm2suba:
+ case Intrinsic::riscv_pm2suba_x:
+ case Intrinsic::riscv_pm2addasu: {
+
+ if (N->getSimpleValueType(0) == MVT::i32) {
+ SDValue Rd = N->getOperand(1);
+ SDValue Rs1 = N->getOperand(2);
+ SDValue Rs2 = N->getOperand(3);
+ SDValue WideRd = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2i32, Rd);
+ SDValue Undef = DAG.getUNDEF(MVT::v2i16);
+ SDValue WideRs1 =
+ DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16, Rs1, Undef);
+ SDValue WideRs2 =
+ DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16, Rs2, Undef);
+ SDValue Res = DAG.getNode(getRVPMulAddAccOpcode(IntNo), DL, MVT::v2i32,
+ WideRd, WideRs1, WideRs2);
+ Results.push_back(DAG.getExtractVectorElt(DL, MVT::i32, Res, 0));
+ return;
+ }
+
+ assert(!Subtarget.is64Bit() && N->getSimpleValueType(0) == MVT::i64 &&
+ "Unexpected multiply-horizontal-add-accumulate legalization");
+ SDValue Rs1 = N->getOperand(2);
+ SDValue Rs2 = N->getOperand(3);
+ auto [RdLo, RdHi] =
+ DAG.SplitScalar(N->getOperand(1), DL, MVT::i32, MVT::i32);
+ SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32);
+ auto Extract = [&](SDValue V, unsigned Idx) {
+ return DAG.getExtractVectorElt(DL, MVT::i32, V, Idx);
+ };
+ SDValue Rs1Lo = Extract(Rs1, 0);
+ SDValue Rs1Hi = Extract(Rs1, 1);
+ SDValue Rs2Lo = Extract(Rs2, 0);
+ SDValue Rs2Hi = Extract(Rs2, 1);
+
+ if (IntNo == Intrinsic::riscv_pmq2adda ||
+ IntNo == Intrinsic::riscv_pmqr2adda) {
+ unsigned AccOpc = IntNo == Intrinsic::riscv_pmq2adda
+ ? RISCVISD::MQWACC
+ : RISCVISD::MQRWACC;
+ SDValue Acc = DAG.getNode(AccOpc, DL, VTs, {RdLo, RdHi, Rs1Lo, Rs2Lo});
+ Acc =
+ DAG.getNode(AccOpc, DL, VTs, {Acc, Acc.getValue(1), Rs1Hi, Rs2Hi});
+ Results.push_back(
+ DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Acc, Acc.getValue(1)));
+ return;
+ }
+
+ if (IntNo == Intrinsic::riscv_pm2suba ||
+ IntNo == Intrinsic::riscv_pm2suba_x) {
+ if (IntNo == Intrinsic::riscv_pm2suba_x)
+ std::swap(Rs2Lo, Rs2Hi);
+ SDValue LoMul = DAG.getNode(ISD::SMUL_LOHI, DL, VTs, Rs1Lo, Rs2Lo);
+ SDValue Acc = DAG.getNode(RISCVISD::ADDD, DL, VTs, RdLo, RdHi, LoMul,
+ LoMul.getValue(1));
+ SDValue HiMul = DAG.getNode(ISD::SMUL_LOHI, DL, VTs, Rs1Hi, Rs2Hi);
+ SDValue Res = DAG.getNode(RISCVISD::SUBD, DL, VTs, Acc, Acc.getValue(1),
+ HiMul, HiMul.getValue(1));
+ Results.push_back(
+ DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Res, Res.getValue(1)));
+ return;
+ }
+
+ if (IntNo == Intrinsic::riscv_pm2adda_x)
+ std::swap(Rs2Lo, Rs2Hi);
+
+ unsigned MulOpc = ISD::SMUL_LOHI;
+ if (IntNo == Intrinsic::riscv_pm2addau)
+ MulOpc = ISD::UMUL_LOHI;
+ else if (IntNo == Intrinsic::riscv_pm2addasu)
+ MulOpc = RISCVISD::WMULSU;
+
+ SDValue LoMul = DAG.getNode(MulOpc, DL, VTs, Rs1Lo, Rs2Lo);
+ SDValue Acc = DAG.getNode(RISCVISD::ADDD, DL, VTs, RdLo, RdHi, LoMul,
+ LoMul.getValue(1));
+ SDValue HiMul = DAG.getNode(MulOpc, DL, VTs, Rs1Hi, Rs2Hi);
+ SDValue Res = DAG.getNode(RISCVISD::ADDD, DL, VTs, Acc, Acc.getValue(1),
+ HiMul, HiMul.getValue(1));
+ Results.push_back(
+ DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Res, Res.getValue(1)));
+ return;
+ }
+
+ case Intrinsic::riscv_pm4adda:
+ case Intrinsic::riscv_pm4addau:
+ case Intrinsic::riscv_pm4addasu: {
+
+ if (N->getSimpleValueType(0) == MVT::i32) {
+ SDValue Rd = N->getOperand(1);
+ SDValue Rs1 = N->getOperand(2);
+ SDValue Rs2 = N->getOperand(3);
+ unsigned Opc = IntNo == Intrinsic::riscv_pm4addau ? RISCVISD::PM4ADDAU
+ : IntNo == Intrinsic::riscv_pm4addasu
+ ? RISCVISD::PM4ADDASU
+ : RISCVISD::PM4ADDA;
+ SDValue WideRd = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2i32, Rd);
+ SDValue Undef = DAG.getUNDEF(MVT::v4i8);
+ SDValue WideRs1 =
+ DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, Rs1, Undef);
+ SDValue WideRs2 =
+ DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, Rs2, Undef);
+ SDValue Res =
+ DAG.getNode(Opc, DL, MVT::v2i32, WideRd, WideRs1, WideRs2);
+ Results.push_back(DAG.getExtractVectorElt(DL, MVT::i32, Res, 0));
+ return;
+ }
+
+ assert(!Subtarget.is64Bit() && N->getSimpleValueType(0) == MVT::i64 &&
+ "Unexpected widening horizontal multiply-accumulate legalization");
+ SDValue Rs1 = N->getOperand(2);
+ SDValue Rs2 = N->getOperand(3);
+ auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
+ auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
+ auto [RdLo, RdHi] =
+ DAG.SplitScalar(N->getOperand(1), DL, MVT::i32, MVT::i32);
+ SDValue Rd = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2i32, RdLo, RdHi);
+ unsigned AccOpc = IntNo == Intrinsic::riscv_pm4addau ? RISCVISD::PM2WADDAU
+ : IntNo == Intrinsic::riscv_pm4addasu
+ ? RISCVISD::PM2WADDASU
+ : RISCVISD::PM2WADDA;
+ SDValue Acc = DAG.getNode(AccOpc, DL, MVT::v2i32, Rd, Rs1Lo, Rs2Lo);
+ Acc = DAG.getNode(AccOpc, DL, MVT::v2i32, Acc, Rs1Hi, Rs2Hi);
+ Results.push_back(DAG.getBitcast(MVT::i64, Acc));
+ return;
+ }
+
case Intrinsic::riscv_pmul_00:
case Intrinsic::riscv_pmul_01:
case Intrinsic::riscv_pmul_11:
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
index 4b192a236d4148..860ab3d3fcad39 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
@@ -2095,6 +2095,19 @@ def riscv_pm4adda : RVSDNode<"PM4ADDA", SDT_RISCVHorizontalMulAcc>;
def riscv_pm4addau : RVSDNode<"PM4ADDAU", SDT_RISCVHorizontalMulAcc>;
def riscv_pm4addasu : RVSDNode<"PM4ADDASU", SDT_RISCVHorizontalMulAcc>;
+// Packed multiplication with horizontal addition and accumulate. Each node
+// is Pat'd at every width its public intrinsic is overloaded at (i32/v2i16,
+// v2i32/v4i16, and i64/v2i32), mirroring how the non-acc riscv_pm2add family
+// is itself Pat'd at all three widths.
+def riscv_pm2adda : RVSDNode<"PM2ADDA", SDT_RISCVHorizontalMulAcc>;
+def riscv_pm2adda_x : RVSDNode<"PM2ADDA_X", SDT_RISCVHorizontalMulAcc>;
+def riscv_pm2addau : RVSDNode<"PM2ADDAU", SDT_RISCVHorizontalMulAcc>;
+def riscv_pmq2adda : RVSDNode<"PMQ2ADDA", SDT_RISCVHorizontalMulAcc>;
+def riscv_pmqr2adda : RVSDNode<"PMQR2ADDA", SDT_RISCVHorizontalMulAcc>;
+def riscv_pm2suba : RVSDNode<"PM2SUBA", SDT_RISCVHorizontalMulAcc>;
+def riscv_pm2suba_x : RVSDNode<"PM2SUBA_X", SDT_RISCVHorizontalMulAcc>;
+def riscv_pm2addasu : RVSDNode<"PM2ADDASU", SDT_RISCVHorizontalMulAcc>;
+
// "Q-format" multiplication
def riscv_mulq : RVSDNode<"MULQ", SDTIntBinOp, [SDNPCommutative]>;
def riscv_mulqr : RVSDNode<"MULQR", SDTIntBinOp, [SDNPCommutative]>;
@@ -2630,6 +2643,30 @@ let append Predicates = [IsRV32] in {
def : PatHorizontalMulAdd<riscv_pm4add, PM4ADDA_B, i32, v4i8>;
def : PatHorizontalMulAdd<riscv_pm4addu, PM4ADDAU_B, i32, v4i8>;
def : PatHorizontalMulAdd<riscv_pm4addsu, PM4ADDASU_B, i32, v4i8>;
+ def : PatHorizontalMulAdd<riscv_pm2add, PM2ADDA_H, i32, v2i16>;
+ def : PatHorizontalMulAdd<riscv_pm2add_x, PM2ADDA_HX, i32, v2i16>;
+ def : PatHorizontalMulAdd<riscv_pm2addu, PM2ADDAU_H, i32, v2i16>;
+ def : PatHorizontalMulAdd<riscv_pmq2add, PMQ2ADDA_H, i32, v2i16>;
+ def : PatHorizontalMulAdd<riscv_pmqr2add, PMQR2ADDA_H, i32, v2i16>;
+ def : PatHorizontalMulAdd<riscv_pm2sub, PM2SUBA_H, i32, v2i16>;
+ def : PatHorizontalMulAdd<riscv_pm2sub_x, PM2SUBA_HX, i32, v2i16>;
+ def : PatHorizontalMulAdd<riscv_pm2addsu, PM2ADDASU_H, i32, v2i16>;
+
+ // Direct ternary-accumulate forms of the above (used by the new
+ // pm2adda/pm2suba/etc. public intrinsics at every width they're
+ // overloaded at; i32/v2i16 here, v2i32/v4i16 and i64/v2i32 further
+ // below in the RV64-only block).
+ def : PatHorizontalMulAcc<riscv_pm2adda, PM2ADDA_H, i32, v2i16>;
+ def : PatHorizontalMulAcc<riscv_pm2adda_x, PM2ADDA_HX, i32, v2i16>;
+ def : PatHorizontalMulAcc<riscv_pm2addau, PM2ADDAU_H, i32, v2i16>;
+ def : PatHorizontalMulAcc<riscv_pmq2adda, PMQ2ADDA_H, i32, v2i16>;
+ def : PatHorizontalMulAcc<riscv_pmqr2adda, PMQR2ADDA_H, i32, v2i16>;
+ def : PatHorizontalMulAcc<riscv_pm2suba, PM2SUBA_H, i32, v2i16>;
+ def : PatHorizontalMulAcc<riscv_pm2suba_x, PM2SUBA_HX, i32, v2i16>;
+ def : PatHorizontalMulAcc<riscv_pm2addasu, PM2ADDASU_H, i32, v2i16>;
+ def : PatHorizontalMulAcc<riscv_pm4adda, PM4ADDA_B, i32, v4i8>;
+ def : PatHorizontalMulAcc<riscv_pm4addau, PM4ADDAU_B, i32, v4i8>;
+ def : PatHorizontalMulAcc<riscv_pm4addasu, PM4ADDASU_B, i32, v4i8>;
def : PatHorizontalMul<riscv_pm4add, PM4ADD_B, i32, v4i8>;
def : PatHorizontalMul<riscv_pm2add, PM2ADD_H, i32, v2i16>;
@@ -3509,6 +3546,9 @@ let append Predicates = [IsRV64] in {
def : PatHorizontalMulAdd<riscv_pm4add, PM4ADDA_B, v2i32, v8i8>;
def : PatHorizontalMulAdd<riscv_pm4addu, PM4ADDAU_B, v2i32, v8i8>;
def : PatHorizontalMulAdd<riscv_pm4addsu, PM4ADDASU_B, v2i32, v8i8>;
+ def : PatHorizontalMulAcc<riscv_pm4adda, PM4ADDA_B, v2i32, v8i8>;
+ def : PatHorizontalMulAcc<riscv_pm4addau, PM4ADDAU_B, v2i32, v8i8>;
+ def : PatHorizontalMulAcc<riscv_pm4addasu, PM4ADDASU_B, v2i32, v8i8>;
def : PatHorizontalMul<riscv_pm4add, PM4ADD_B, v2i32, v8i8>;
def : PatHorizontalMul<riscv_pm2add, PM2ADD_H, v2i32, v4i16>;
@@ -3539,6 +3579,29 @@ let append Predicates = [IsRV64] in {
def : PatHorizontalMulAcc<riscv_pm4addau, PM4ADDAU_H, i64, v4i16>;
def : PatHorizontalMulAcc<riscv_pm4addasu, PM4ADDASU_H, i64, v4i16>;
+ // Direct ternary-accumulate forms of pm2adda/pm2suba/etc. at v2i32/v4i16
+ // width (the "_i16x4" intrinsics; pm2adda.h etc. compute two independent
+ // 32-bit-group results in parallel on RV64, matching this width exactly).
+ def : PatHorizontalMulAcc<riscv_pm2adda, PM2ADDA_H, v2i32, v4i16>;
+ def : PatHorizontalMulAcc<riscv_pm2adda_x, PM2ADDA_HX, v2i32, v4i16>;
+ def : PatHorizontalMulAcc<riscv_pm2addau, PM2ADDAU_H, v2i32, v4i16>;
+ def : PatHorizontalMulAcc<riscv_pmq2adda, PMQ2ADDA_H, v2i32, v4i16>;
+ def : PatHorizontalMulAcc<riscv_pmqr2adda, PMQR2ADDA_H, v2i32, v4i16>;
+ def : PatHorizontalMulAcc<riscv_pm2suba, PM2SUBA_H, v2i32, v4i16>;
+ def : PatHorizontalMulAcc<riscv_pm2suba_x, PM2SUBA_HX, v2i32, v4i16>;
+ def : PatHorizontalMulAcc<riscv_pm2addasu, PM2ADDASU_H, v2i32, v4i16>;
+
+ // Direct ternary-accumulate forms at i64/v2i32 width (the "_i32x2"
+ // intrinsics), pairing with the W-suffixed instructions.
+ def : PatHorizontalMulAcc<riscv_pm2adda, PM2ADDA_W, i64, v2i32>;
+ def : PatHorizontalMulAcc<riscv_pm2adda_x, PM2ADDA_WX, i64, v2i32>;
+ def : PatHorizontalMulAcc<riscv_pm2addau, PM2ADDAU_W, i64, v2i32>;
+ def : PatHorizontalMulAcc<riscv_pmq2adda, PMQ2ADDA_W, i64, v2i32>;
+ def : PatHorizontalMulAcc<riscv_pmqr2adda, PMQR2ADDA_W, i64, v2i32>;
+ def : PatHorizontalMulAcc<riscv_pm2suba, PM2SUBA_W, i64, v2i32>;
+ def : PatHorizontalMulAcc<riscv_pm2suba_x, PM2SUBA_WX, i64, v2i32>;
+ def : PatHorizontalMulAcc<riscv_pm2addasu, PM2ADDASU_W, i64, v2i32>;
+
// 8/16/32-bit multiply low patterns
// FIXME custom lower
def : Pat<(v8i8 (mul GPR:$rs1, GPR:$rs2)),
diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
index 45d3e90405f4c2..fc5024a5b5f674 100644
--- a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
+++ b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
@@ -9288,3 +9288,392 @@ define <2 x i32> @test_pslidedownx_v2i32(<2 x i32> %rd, <2 x i32> %rs1,
%res = bitcast i64 %res.bits to <2 x i32>
ret <2 x i32> %res
}
+
+; Packed Multiplication with Horizontal Addition and Accumulate.
+define i32 @test_pm4adda_v4i8_i32(i32 %rd, <4 x i8> %a, <4 x i8> %b) {
+; CHECK-LABEL: test_pm4adda_v4i8_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm4adda.b a0, a1, a2
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm4adda.i32.v4i8(i32 %rd, <4 x i8> %a, <4 x i8> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm4addau_v4i8_i32(i32 %rd, <4 x i8> %a, <4 x i8> %b) {
+; CHECK-LABEL: test_pm4addau_v4i8_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm4addau.b a0, a1, a2
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm4addau.i32.v4i8(i32 %rd, <4 x i8> %a, <4 x i8> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm4addasu_v4i8_i32(i32 %rd, <4 x i8> %a, <4 x i8> %b) {
+; CHECK-LABEL: test_pm4addasu_v4i8_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm4addasu.b a0, a1, a2
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm4addasu.i32.v4i8(i32 %rd, <4 x i8> %a, <4 x i8> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2adda_v2i16_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2adda_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2adda.h a0, a1, a2
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2adda.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2adda_x_v2i16_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2adda_x_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2adda.hx a0, a1, a2
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2adda.x.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2addau_v2i16_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2addau_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2addau.h a0, a1, a2
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2addau.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pmq2adda_v2i16_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pmq2adda_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pmq2adda.h a0, a1, a2
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pmq2adda.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pmqr2adda_v2i16_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pmqr2adda_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pmqr2adda.h a0, a1, a2
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pmqr2adda.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2suba_v2i16_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2suba_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2suba.h a0, a1, a2
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2suba.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2suba_x_v2i16_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2suba_x_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2suba.hx a0, a1, a2
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2suba.x.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_pm2addasu_v2i16_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) {
+; CHECK-LABEL: test_pm2addasu_v2i16_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: pm2addasu.h a0, a1, a2
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.pm2addasu.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define <2 x i32> @test_pm2adda_v4i16_v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2adda_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2adda.h a1, a3, a5
+; RV32-NEXT: pm2adda.h a0, a2, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2adda_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2adda.h a0, a1, a2
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2adda.v2i32.v4i16(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm2adda_x_v4i16_v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2adda_x_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2adda.hx a1, a3, a5
+; RV32-NEXT: pm2adda.hx a0, a2, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2adda_x_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2adda.hx a0, a1, a2
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2adda.x.v2i32.v4i16(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm2addau_v4i16_v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2addau_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2addau.h a1, a3, a5
+; RV32-NEXT: pm2addau.h a0, a2, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2addau_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2addau.h a0, a1, a2
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2addau.v2i32.v4i16(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pmq2adda_v4i16_v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pmq2adda_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pmq2adda.h a1, a3, a5
+; RV32-NEXT: pmq2adda.h a0, a2, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pmq2adda_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pmq2adda.h a0, a1, a2
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pmq2adda.v2i32.v4i16(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pmqr2adda_v4i16_v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pmqr2adda_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pmqr2adda.h a1, a3, a5
+; RV32-NEXT: pmqr2adda.h a0, a2, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pmqr2adda_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pmqr2adda.h a0, a1, a2
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pmqr2adda.v2i32.v4i16(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm2suba_v4i16_v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2suba_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2suba.h a1, a3, a5
+; RV32-NEXT: pm2suba.h a0, a2, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2suba_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2suba.h a0, a1, a2
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2suba.v2i32.v4i16(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm2suba_x_v4i16_v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2suba_x_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2suba.hx a1, a3, a5
+; RV32-NEXT: pm2suba.hx a0, a2, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2suba_x_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2suba.hx a0, a1, a2
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2suba.x.v2i32.v4i16(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pm2addasu_v4i16_v2i32(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm2addasu_v4i16_v2i32:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2addasu.h a1, a3, a5
+; RV32-NEXT: pm2addasu.h a0, a2, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2addasu_v4i16_v2i32:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2addasu.h a0, a1, a2
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm2addasu.v2i32.v4i16(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define i64 @test_pm2adda_v2i32_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pm2adda_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: wmacc a0, a2, a4
+; RV32-NEXT: wmacc a0, a3, a5
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2adda_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2adda.w a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2adda.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm2adda_x_v2i32_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pm2adda_x_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: wmacc a0, a2, a5
+; RV32-NEXT: wmacc a0, a3, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2adda_x_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2adda.wx a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2adda.x.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm2addau_v2i32_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pm2addau_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: wmaccu a0, a2, a4
+; RV32-NEXT: wmaccu a0, a3, a5
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2addau_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2addau.w a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2addau.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pmq2adda_v2i32_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pmq2adda_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mqwacc a0, a2, a4
+; RV32-NEXT: mqwacc a0, a3, a5
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pmq2adda_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pmq2adda.w a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pmq2adda.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pmqr2adda_v2i32_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pmqr2adda_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mqrwacc a0, a2, a4
+; RV32-NEXT: mqrwacc a0, a3, a5
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pmqr2adda_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pmqr2adda.w a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pmqr2adda.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm2suba_v2i32_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pm2suba_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: wmacc a0, a2, a4
+; RV32-NEXT: wmul a2, a3, a5
+; RV32-NEXT: subd a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2suba_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2suba.w a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2suba.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm2suba_x_v2i32_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pm2suba_x_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: wmacc a0, a2, a5
+; RV32-NEXT: wmul a2, a3, a4
+; RV32-NEXT: subd a0, a0, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2suba_x_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2suba.wx a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2suba.x.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm2addasu_v2i32_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_pm2addasu_v2i32_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: wmaccsu a0, a2, a4
+; RV32-NEXT: wmaccsu a0, a3, a5
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm2addasu_v2i32_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm2addasu.w a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm2addasu.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm4adda_v4i16_i64(i64 %rd, <4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm4adda_v4i16_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2wadda.h a0, a2, a4
+; RV32-NEXT: pm2wadda.h a0, a3, a5
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm4adda_v4i16_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm4adda.h a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm4adda.i64.v4i16(i64 %rd, <4 x i16> %a, <4 x i16> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm4addau_v4i16_i64(i64 %rd, <4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm4addau_v4i16_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2waddau.h a0, a2, a4
+; RV32-NEXT: pm2waddau.h a0, a3, a5
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm4addau_v4i16_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm4addau.h a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm4addau.i64.v4i16(i64 %rd, <4 x i16> %a, <4 x i16> %b)
+ ret i64 %r
+}
+
+define i64 @test_pm4addasu_v4i16_i64(i64 %rd, <4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pm4addasu_v4i16_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: pm2waddasu.h a0, a2, a4
+; RV32-NEXT: pm2waddasu.h a0, a3, a5
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pm4addasu_v4i16_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: pm4addasu.h a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.pm4addasu.i64.v4i16(i64 %rd, <4 x i16> %a, <4 x i16> %b)
+ ret i64 %r
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
>From 0a198305e333eff4ae9f5a7df72acd2fc5594e68 Mon Sep 17 00:00:00 2001
From: Dongyan Chen <chendongyan at isrc.iscas.ac.cn>
Date: Fri, 30 Oct 2026 05:31:29 +0000
Subject: [PATCH 2/5] [RISCV][P-ext] Address review: drop "new" from comment,
clarify i32 reachability
---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 7 +++++--
llvm/lib/Target/RISCV/RISCVInstrInfoP.td | 6 +++---
2 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 329411a956deff..ea202a38bddb4c 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -13875,7 +13875,6 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
case Intrinsic::riscv_pm2suba:
case Intrinsic::riscv_pm2suba_x:
case Intrinsic::riscv_pm2addasu: {
-
MVT VT = Op.getSimpleValueType();
SDValue Rd = Op.getOperand(1);
SDValue Rs1 = Op.getOperand(2);
@@ -13905,7 +13904,6 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
case Intrinsic::riscv_pm4adda:
case Intrinsic::riscv_pm4addau:
case Intrinsic::riscv_pm4addasu: {
-
MVT VT = Op.getSimpleValueType();
SDValue Rd = Op.getOperand(1);
SDValue Rs1 = Op.getOperand(2);
@@ -13914,6 +13912,11 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
: IntNo == Intrinsic::riscv_pm4addasu ? RISCVISD::PM4ADDASU
: RISCVISD::PM4ADDA;
+ // i32 (the "_i8x4" width) is reachable on both RV32 and RV64: it's the
+ // XLenVT on RV32, and on RV64 ISD::INTRINSIC_WO_CHAIN is registered
+ // Custom for i32 results (see the is64Bit() setOperationAction above),
+ // so this intrinsic's i32 result still reaches LowerOperation there too
+ // -- the generic legalizer appends the ABI-required sext.w afterward.
if (VT == MVT::i32)
return DAG.getNode(Opc, DL, MVT::i32, Rd, Rs1, Rs2);
assert(Subtarget.is64Bit() && VT == MVT::i64 &&
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
index 860ab3d3fcad39..d7a35acef64567 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
@@ -2652,10 +2652,10 @@ let append Predicates = [IsRV32] in {
def : PatHorizontalMulAdd<riscv_pm2sub_x, PM2SUBA_HX, i32, v2i16>;
def : PatHorizontalMulAdd<riscv_pm2addsu, PM2ADDASU_H, i32, v2i16>;
- // Direct ternary-accumulate forms of the above (used by the new
+ // Direct ternary-accumulate forms of the above, used by the
// pm2adda/pm2suba/etc. public intrinsics at every width they're
- // overloaded at; i32/v2i16 here, v2i32/v4i16 and i64/v2i32 further
- // below in the RV64-only block).
+ // overloaded at: i32/v2i16 here, v2i32/v4i16 and i64/v2i32 further
+ // below in the RV64-only block.
def : PatHorizontalMulAcc<riscv_pm2adda, PM2ADDA_H, i32, v2i16>;
def : PatHorizontalMulAcc<riscv_pm2adda_x, PM2ADDA_HX, i32, v2i16>;
def : PatHorizontalMulAcc<riscv_pm2addau, PM2ADDAU_H, i32, v2i16>;
>From 7283022399c1eaeb6bcd86bb21fb7c46ec0a3a8a Mon Sep 17 00:00:00 2001
From: Dongyan Chen <chendongyan at isrc.iscas.ac.cn>
Date: Mon, 2 Nov 2026 22:08:36 +0800
Subject: [PATCH 3/5] Assert XLenVT for pm4adda results in LowerOperation
---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 14 +++++---------
1 file changed, 5 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index ea202a38bddb4c..3db89f0f9ed3c1 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -13912,16 +13912,12 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
: IntNo == Intrinsic::riscv_pm4addasu ? RISCVISD::PM4ADDASU
: RISCVISD::PM4ADDA;
- // i32 (the "_i8x4" width) is reachable on both RV32 and RV64: it's the
- // XLenVT on RV32, and on RV64 ISD::INTRINSIC_WO_CHAIN is registered
- // Custom for i32 results (see the is64Bit() setOperationAction above),
- // so this intrinsic's i32 result still reaches LowerOperation there too
- // -- the generic legalizer appends the ABI-required sext.w afterward.
- if (VT == MVT::i32)
- return DAG.getNode(Opc, DL, MVT::i32, Rd, Rs1, Rs2);
- assert(Subtarget.is64Bit() && VT == MVT::i64 &&
+ // An i32 result is illegal on RV64, so it is type-promoted and handled by
+ // ReplaceNodeResults instead. Only XLenVT (i32 on RV32, i64 on RV64)
+ // reaches LowerOperation.
+ assert(VT == Subtarget.getXLenVT() &&
"Unexpected packed multiply-horizontal-add-accumulate result");
- return DAG.getNode(Opc, DL, MVT::i64, Rd, Rs1, Rs2);
+ return DAG.getNode(Opc, DL, VT, Rd, Rs1, Rs2);
}
case Intrinsic::riscv_pmerge: {
EVT VT = Op.getValueType();
>From 0af9d4ad7f6ded72e53af0d469b57ef7eaf6df59 Mon Sep 17 00:00:00 2001
From: Dongyan Chen <chendongyan at isrc.iscas.ac.cn>
Date: Tue, 3 Nov 2026 12:22:55 +0800
Subject: [PATCH 4/5] Handle v2i32 results for packed horizontal
multiply-accumulate
---
clang/lib/Headers/riscv_packed_simd.h | 3 +-
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 64 +++++++------------
.../CodeGen/RISCV/rvp-byte-horizontal-acc.ll | 51 +++++++++++++++
3 files changed, 76 insertions(+), 42 deletions(-)
diff --git a/clang/lib/Headers/riscv_packed_simd.h b/clang/lib/Headers/riscv_packed_simd.h
index 4f3af7d39dc520..02862aeca1b981 100644
--- a/clang/lib/Headers/riscv_packed_simd.h
+++ b/clang/lib/Headers/riscv_packed_simd.h
@@ -1233,8 +1233,7 @@ __packed_ternary_builtin_mixed(pm2suba_i16x2, int32_t, int16x2_t, int16x2_t, __b
__packed_ternary_builtin_mixed(pm2suba_x_i16x2, int32_t, int16x2_t, int16x2_t, __builtin_riscv_pm2suba_x_i16x2)
__packed_ternary_builtin_mixed(pm2addasu_i16x2, int32_t, int16x2_t, uint16x2_t, __builtin_riscv_pm2addasu_i16x2)
-
- /* Packed Multiplication with Horizontal Addition and Accumulate (64-bit) */
+/* Packed Multiplication with Horizontal Addition and Accumulate (64-bit) */
static __inline__ int32x2_t __DEFAULT_FN_ATTRS
__riscv_pm4adda_i8x8(int32x2_t __rd, int8x8_t __rs1, int8x8_t __rs2) {
return (int32x2_t)((uint32x2_t)__rd +
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 3db89f0f9ed3c1..8023dd3159cf24 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -12707,6 +12707,12 @@ static unsigned getRVPMulAddAccOpcode(unsigned IntNo) {
return RISCVISD::PM2SUBA_X;
case Intrinsic::riscv_pm2addasu:
return RISCVISD::PM2ADDASU;
+ case Intrinsic::riscv_pm4adda:
+ return RISCVISD::PM4ADDA;
+ case Intrinsic::riscv_pm4addau:
+ return RISCVISD::PM4ADDAU;
+ case Intrinsic::riscv_pm4addasu:
+ return RISCVISD::PM4ADDASU;
}
}
@@ -13874,33 +13880,7 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
case Intrinsic::riscv_pmqr2adda:
case Intrinsic::riscv_pm2suba:
case Intrinsic::riscv_pm2suba_x:
- case Intrinsic::riscv_pm2addasu: {
- MVT VT = Op.getSimpleValueType();
- SDValue Rd = Op.getOperand(1);
- SDValue Rs1 = Op.getOperand(2);
- SDValue Rs2 = Op.getOperand(3);
- unsigned Opc = getRVPMulAddAccOpcode(IntNo);
-
- if (VT == MVT::i32)
- return DAG.getNode(Opc, DL, MVT::i32, Rd, Rs1, Rs2);
-
- if (VT == MVT::v2i32) {
- if (Subtarget.is64Bit())
- return DAG.getNode(Opc, DL, MVT::v2i32, Rd, Rs1, Rs2);
-
- auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
- auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
- SDValue RdLo = DAG.getExtractVectorElt(DL, MVT::i32, Rd, 0);
- SDValue RdHi = DAG.getExtractVectorElt(DL, MVT::i32, Rd, 1);
- SDValue Lo = DAG.getNode(Opc, DL, MVT::i32, RdLo, Rs1Lo, Rs2Lo);
- SDValue Hi = DAG.getNode(Opc, DL, MVT::i32, RdHi, Rs1Hi, Rs2Hi);
- return DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2i32, Lo, Hi);
- }
-
- assert(Subtarget.is64Bit() && VT == MVT::i64 &&
- "Unexpected packed multiply-horizontal-add-accumulate result");
- return DAG.getNode(Opc, DL, MVT::i64, Rd, Rs1, Rs2);
- }
+ case Intrinsic::riscv_pm2addasu:
case Intrinsic::riscv_pm4adda:
case Intrinsic::riscv_pm4addau:
case Intrinsic::riscv_pm4addasu: {
@@ -13908,16 +13888,23 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
SDValue Rd = Op.getOperand(1);
SDValue Rs1 = Op.getOperand(2);
SDValue Rs2 = Op.getOperand(3);
- unsigned Opc = IntNo == Intrinsic::riscv_pm4addau ? RISCVISD::PM4ADDAU
- : IntNo == Intrinsic::riscv_pm4addasu ? RISCVISD::PM4ADDASU
- : RISCVISD::PM4ADDA;
-
- // An i32 result is illegal on RV64, so it is type-promoted and handled by
- // ReplaceNodeResults instead. Only XLenVT (i32 on RV32, i64 on RV64)
- // reaches LowerOperation.
- assert(VT == Subtarget.getXLenVT() &&
+ unsigned Opc = getRVPMulAddAccOpcode(IntNo);
+
+ if (VT == Subtarget.getXLenVT())
+ return DAG.getNode(Opc, DL, VT, Rd, Rs1, Rs2);
+
+ assert(VT == MVT::v2i32 &&
"Unexpected packed multiply-horizontal-add-accumulate result");
- return DAG.getNode(Opc, DL, VT, Rd, Rs1, Rs2);
+ if (Subtarget.is64Bit())
+ return DAG.getNode(Opc, DL, VT, Rd, Rs1, Rs2);
+
+ auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
+ auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
+ SDValue RdLo = DAG.getExtractVectorElt(DL, MVT::i32, Rd, 0);
+ SDValue RdHi = DAG.getExtractVectorElt(DL, MVT::i32, Rd, 1);
+ SDValue Lo = DAG.getNode(Opc, DL, MVT::i32, RdLo, Rs1Lo, Rs2Lo);
+ SDValue Hi = DAG.getNode(Opc, DL, MVT::i32, RdHi, Rs1Hi, Rs2Hi);
+ return DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2i32, Lo, Hi);
}
case Intrinsic::riscv_pmerge: {
EVT VT = Op.getValueType();
@@ -18255,10 +18242,7 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
SDValue Rd = N->getOperand(1);
SDValue Rs1 = N->getOperand(2);
SDValue Rs2 = N->getOperand(3);
- unsigned Opc = IntNo == Intrinsic::riscv_pm4addau ? RISCVISD::PM4ADDAU
- : IntNo == Intrinsic::riscv_pm4addasu
- ? RISCVISD::PM4ADDASU
- : RISCVISD::PM4ADDA;
+ unsigned Opc = getRVPMulAddAccOpcode(IntNo);
SDValue WideRd = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2i32, Rd);
SDValue Undef = DAG.getUNDEF(MVT::v4i8);
SDValue WideRs1 =
diff --git a/llvm/test/CodeGen/RISCV/rvp-byte-horizontal-acc.ll b/llvm/test/CodeGen/RISCV/rvp-byte-horizontal-acc.ll
index 505cfd7ed8475e..2b286d723fc111 100644
--- a/llvm/test/CodeGen/RISCV/rvp-byte-horizontal-acc.ll
+++ b/llvm/test/CodeGen/RISCV/rvp-byte-horizontal-acc.ll
@@ -129,3 +129,54 @@ define <2 x i32> @pm4adda_shared_words(<2 x i32> %rd, <8 x i8> %a, <8 x i8> %b,
%r = add <2 x i32> %rd, %dot
ret <2 x i32> %r
}
+
+define <2 x i32> @pm4adda_acc(<2 x i32> %rd, <8 x i8> %a, <8 x i8> %b) {
+; RV32-LABEL: pm4adda_acc:
+; RV32: # %bb.0:
+; RV32-NEXT: pm4adda.b a1, a3, a5
+; RV32-NEXT: pm4adda.b a0, a2, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: pm4adda_acc:
+; RV64: # %bb.0:
+; RV64-NEXT: pm4adda.b a0, a1, a2
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm4adda.v2i32.v8i8(<2 x i32> %rd, <8 x i8> %a, <8 x i8> %b)
+ ret <2 x i32> %r
+}
+
+declare <2 x i32> @llvm.riscv.pm4adda.v2i32.v8i8(<2 x i32>, <8 x i8>, <8 x i8>)
+
+define <2 x i32> @pm4addau_acc(<2 x i32> %rd, <8 x i8> %a, <8 x i8> %b) {
+; RV32-LABEL: pm4addau_acc:
+; RV32: # %bb.0:
+; RV32-NEXT: pm4addau.b a1, a3, a5
+; RV32-NEXT: pm4addau.b a0, a2, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: pm4addau_acc:
+; RV64: # %bb.0:
+; RV64-NEXT: pm4addau.b a0, a1, a2
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm4addau.v2i32.v8i8(<2 x i32> %rd, <8 x i8> %a, <8 x i8> %b)
+ ret <2 x i32> %r
+}
+
+declare <2 x i32> @llvm.riscv.pm4addau.v2i32.v8i8(<2 x i32>, <8 x i8>, <8 x i8>)
+
+define <2 x i32> @pm4addasu_acc(<2 x i32> %rd, <8 x i8> %a, <8 x i8> %b) {
+; RV32-LABEL: pm4addasu_acc:
+; RV32: # %bb.0:
+; RV32-NEXT: pm4addasu.b a1, a3, a5
+; RV32-NEXT: pm4addasu.b a0, a2, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: pm4addasu_acc:
+; RV64: # %bb.0:
+; RV64-NEXT: pm4addasu.b a0, a1, a2
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pm4addasu.v2i32.v8i8(<2 x i32> %rd, <8 x i8> %a, <8 x i8> %b)
+ ret <2 x i32> %r
+}
+
+declare <2 x i32> @llvm.riscv.pm4addasu.v2i32.v8i8(<2 x i32>, <8 x i8>, <8 x i8>)
>From dd4b467e6c1e9b3444b0645565358ccb5a365559 Mon Sep 17 00:00:00 2001
From: Dongyan Chen <chendongyan at isrc.iscas.ac.cn>
Date: Tue, 3 Nov 2026 14:46:33 +0800
Subject: [PATCH 5/5] add dedicated builtins for pm4add**8x8
---
clang/include/clang/Basic/BuiltinsRISCV.td | 3 +++
clang/lib/CodeGen/TargetBuiltins/RISCV.cpp | 6 ++++++
clang/lib/Headers/riscv_packed_simd.h | 20 +++-----------------
clang/test/CodeGen/RISCV/rvp-intrinsics.c | 15 ++++++---------
4 files changed, 18 insertions(+), 26 deletions(-)
diff --git a/clang/include/clang/Basic/BuiltinsRISCV.td b/clang/include/clang/Basic/BuiltinsRISCV.td
index 5f10c8dfd49210..19438ec8d6b0bb 100644
--- a/clang/include/clang/Basic/BuiltinsRISCV.td
+++ b/clang/include/clang/Basic/BuiltinsRISCV.td
@@ -422,6 +422,9 @@ def pm2suba_x_i16x2 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short
def pm2addasu_i16x2 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, unsigned short>)">;
// Packed Multiplication with Horizontal Addition and Accumulate (64-bit)
+def pm4adda_i8x8 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<8, signed char>, _Vector<8, signed char>)">;
+def pm4addau_u8x8 : RISCVBuiltin<"_Vector<2, unsigned int>(_Vector<2, unsigned int>, _Vector<8, unsigned char>, _Vector<8, unsigned char>)">;
+def pm4addasu_i8x8 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<8, signed char>, _Vector<8, unsigned char>)">;
def pm2adda_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">;
def pm2adda_x_i16x4 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">;
def pm2addau_u16x4 : RISCVBuiltin<"_Vector<2, unsigned int>(_Vector<2, unsigned int>, _Vector<4, unsigned short>, _Vector<4, unsigned short>)">;
diff --git a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
index e61f8950b51bac..8cf84658ab57ee 100644
--- a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
@@ -1994,10 +1994,13 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID,
case RISCV::BI__builtin_riscv_pm2addasu_i16x4:
case RISCV::BI__builtin_riscv_pm2addasu_i32x2:
case RISCV::BI__builtin_riscv_pm4adda_i8x4:
+ case RISCV::BI__builtin_riscv_pm4adda_i8x8:
case RISCV::BI__builtin_riscv_pm4adda_i16x4:
case RISCV::BI__builtin_riscv_pm4addau_u8x4:
+ case RISCV::BI__builtin_riscv_pm4addau_u8x8:
case RISCV::BI__builtin_riscv_pm4addau_u16x4:
case RISCV::BI__builtin_riscv_pm4addasu_i8x4:
+ case RISCV::BI__builtin_riscv_pm4addasu_i8x8:
case RISCV::BI__builtin_riscv_pm4addasu_i16x4: {
switch (BuiltinID) {
default:
@@ -2043,14 +2046,17 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID,
ID = Intrinsic::riscv_pm2addasu;
break;
case RISCV::BI__builtin_riscv_pm4adda_i8x4:
+ case RISCV::BI__builtin_riscv_pm4adda_i8x8:
case RISCV::BI__builtin_riscv_pm4adda_i16x4:
ID = Intrinsic::riscv_pm4adda;
break;
case RISCV::BI__builtin_riscv_pm4addau_u8x4:
+ case RISCV::BI__builtin_riscv_pm4addau_u8x8:
case RISCV::BI__builtin_riscv_pm4addau_u16x4:
ID = Intrinsic::riscv_pm4addau;
break;
case RISCV::BI__builtin_riscv_pm4addasu_i8x4:
+ case RISCV::BI__builtin_riscv_pm4addasu_i8x8:
case RISCV::BI__builtin_riscv_pm4addasu_i16x4:
ID = Intrinsic::riscv_pm4addasu;
break;
diff --git a/clang/lib/Headers/riscv_packed_simd.h b/clang/lib/Headers/riscv_packed_simd.h
index 02862aeca1b981..bb71abd82ed74c 100644
--- a/clang/lib/Headers/riscv_packed_simd.h
+++ b/clang/lib/Headers/riscv_packed_simd.h
@@ -1234,23 +1234,9 @@ __packed_ternary_builtin_mixed(pm2suba_x_i16x2, int32_t, int16x2_t, int16x2_t, _
__packed_ternary_builtin_mixed(pm2addasu_i16x2, int32_t, int16x2_t, uint16x2_t, __builtin_riscv_pm2addasu_i16x2)
/* Packed Multiplication with Horizontal Addition and Accumulate (64-bit) */
-static __inline__ int32x2_t __DEFAULT_FN_ATTRS
-__riscv_pm4adda_i8x8(int32x2_t __rd, int8x8_t __rs1, int8x8_t __rs2) {
- return (int32x2_t)((uint32x2_t)__rd +
- (uint32x2_t)__riscv_pm4add_i8x8(__rs1, __rs2));
-}
-
-static __inline__ uint32x2_t __DEFAULT_FN_ATTRS
-__riscv_pm4addau_u8x8(uint32x2_t __rd, uint8x8_t __rs1, uint8x8_t __rs2) {
- return __rd + __riscv_pm4addu_u8x8(__rs1, __rs2);
-}
-
-static __inline__ int32x2_t __DEFAULT_FN_ATTRS
-__riscv_pm4addasu_i8x8(int32x2_t __rd, int8x8_t __rs1, uint8x8_t __rs2) {
- return (int32x2_t)((uint32x2_t)__rd +
- (uint32x2_t)__riscv_pm4addsu_i8x8(__rs1, __rs2));
-}
-
+__packed_ternary_builtin_mixed(pm4adda_i8x8, int32x2_t, int8x8_t, int8x8_t, __builtin_riscv_pm4adda_i8x8)
+__packed_ternary_builtin_mixed(pm4addau_u8x8, uint32x2_t, uint8x8_t, uint8x8_t, __builtin_riscv_pm4addau_u8x8)
+__packed_ternary_builtin_mixed(pm4addasu_i8x8, int32x2_t, int8x8_t, uint8x8_t, __builtin_riscv_pm4addasu_i8x8)
__packed_ternary_builtin_mixed(pm2adda_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2adda_i16x4)
__packed_ternary_builtin_mixed(pm2adda_x_i16x4, int32x2_t, int16x4_t, int16x4_t, __builtin_riscv_pm2adda_x_i16x4)
__packed_ternary_builtin_mixed(pm2addau_u16x4, uint32x2_t, uint16x4_t, uint16x4_t, __builtin_riscv_pm2addau_u16x4)
diff --git a/clang/test/CodeGen/RISCV/rvp-intrinsics.c b/clang/test/CodeGen/RISCV/rvp-intrinsics.c
index 598db5ca5863c2..86a9a6cf3372d6 100644
--- a/clang/test/CodeGen/RISCV/rvp-intrinsics.c
+++ b/clang/test/CodeGen/RISCV/rvp-intrinsics.c
@@ -10837,9 +10837,8 @@ uint32x2_t test_pslidedownx_u32x2(uint32x2_t rd, uint32x2_t rs1,
// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[A_COERCE]] to <8 x i8>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast i64 [[B_COERCE]] to <8 x i8>
-// CHECK-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pm4add.v2i32.v8i8(<8 x i8> [[TMP1]], <8 x i8> [[TMP2]])
-// CHECK-NEXT: [[ADD_I:%.*]] = add <2 x i32> [[TMP3]], [[TMP0]]
-// CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[ADD_I]] to i64
+// CHECK-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pm4adda.v2i32.v8i8(<2 x i32> [[TMP0]], <8 x i8> [[TMP1]], <8 x i8> [[TMP2]])
+// CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
// CHECK-NEXT: ret i64 [[TMP4]]
//
int32x2_t test_pm4adda_i8x8(int32x2_t rd, int8x8_t a, int8x8_t b) {
@@ -10852,9 +10851,8 @@ int32x2_t test_pm4adda_i8x8(int32x2_t rd, int8x8_t a, int8x8_t b) {
// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[A_COERCE]] to <8 x i8>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast i64 [[B_COERCE]] to <8 x i8>
-// CHECK-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pm4addu.v2i32.v8i8(<8 x i8> [[TMP1]], <8 x i8> [[TMP2]])
-// CHECK-NEXT: [[ADD_I:%.*]] = add <2 x i32> [[TMP3]], [[TMP0]]
-// CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[ADD_I]] to i64
+// CHECK-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pm4addau.v2i32.v8i8(<2 x i32> [[TMP0]], <8 x i8> [[TMP1]], <8 x i8> [[TMP2]])
+// CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
// CHECK-NEXT: ret i64 [[TMP4]]
//
uint32x2_t test_pm4addau_u8x8(uint32x2_t rd, uint8x8_t a, uint8x8_t b) {
@@ -10867,9 +10865,8 @@ uint32x2_t test_pm4addau_u8x8(uint32x2_t rd, uint8x8_t a, uint8x8_t b) {
// CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
// CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[A_COERCE]] to <8 x i8>
// CHECK-NEXT: [[TMP2:%.*]] = bitcast i64 [[B_COERCE]] to <8 x i8>
-// CHECK-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pm4addsu.v2i32.v8i8(<8 x i8> [[TMP1]], <8 x i8> [[TMP2]])
-// CHECK-NEXT: [[ADD_I:%.*]] = add <2 x i32> [[TMP3]], [[TMP0]]
-// CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[ADD_I]] to i64
+// CHECK-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pm4addasu.v2i32.v8i8(<2 x i32> [[TMP0]], <8 x i8> [[TMP1]], <8 x i8> [[TMP2]])
+// CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
// CHECK-NEXT: ret i64 [[TMP4]]
//
int32x2_t test_pm4addasu_i8x8(int32x2_t rd, int8x8_t a, uint8x8_t b) {
More information about the cfe-commits
mailing list