[clang] [llvm] [RISCV][P-ext] Support Packed "Q-format" Multiply Parts Accumulate (PR #217918)
via cfe-commits
cfe-commits at lists.llvm.org
Mon Aug 24 20:00:43 PDT 2026
https://github.com/TelGome updated https://github.com/llvm/llvm-project/pull/217918
>From 9a2e629cea49b1ec6a4405771f7919279d420d70 Mon Sep 17 00:00:00 2001
From: Dongyan Chen <chendongyan at isrc.iscas.ac.cn>
Date: Fri, 21 Aug 2026 13:05:05 +0000
Subject: [PATCH 1/2] [RISCV][P-ext] Support Packed "Q-format" Multiply Parts
Accumulate
---
clang/include/clang/Basic/BuiltinsRISCV.td | 22 +
clang/lib/CodeGen/TargetBuiltins/RISCV.cpp | 82 ++++
clang/lib/Headers/riscv_packed_simd.h | 32 +-
clang/test/CodeGen/RISCV/rvp-intrinsics.c | 388 ++++++++++++++++++
.../riscv_packed_simd.c | 126 ++++++
llvm/include/llvm/IR/IntrinsicsRISCV.td | 25 ++
llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp | 16 +
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 239 ++++++++++-
llvm/lib/Target/RISCV/RISCVInstrInfoP.td | 122 ++++++
llvm/test/CodeGen/RISCV/rvp-simd-32.ll | 96 +++++
llvm/test/CodeGen/RISCV/rvp-simd-64.ll | 177 ++++++++
11 files changed, 1312 insertions(+), 13 deletions(-)
diff --git a/clang/include/clang/Basic/BuiltinsRISCV.td b/clang/include/clang/Basic/BuiltinsRISCV.td
index 89dbf736f4435..b292ffc0140e8 100644
--- a/clang/include/clang/Basic/BuiltinsRISCV.td
+++ b/clang/include/clang/Basic/BuiltinsRISCV.td
@@ -319,6 +319,28 @@ def pmulqr_i16x4 : RISCVBuiltin<"_Vector<4, short>(_Vector<4, short>, _Vector<4,
def pmulq_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<2, int>)">;
def pmulqr_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<2, int>)">;
+// Packed "Q-format" Multiply Parts Accumulate (32-bit)
+def mqacc_h00_i32 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short>)">;
+def mqacc_h01_i32 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short>)">;
+def mqacc_h11_i32 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short>)">;
+def mqracc_h00_i32 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short>)">;
+def mqracc_h01_i32 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short>)">;
+def mqracc_h11_i32 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short>)">;
+
+// Packed "Q-format" Multiply Parts Accumulate (64-bit)
+def pmqacc_h00_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">;
+def pmqacc_h01_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">;
+def pmqacc_h11_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">;
+def pmqracc_h00_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">;
+def pmqracc_h01_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">;
+def pmqracc_h11_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">;
+def mqacc_w00_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">;
+def mqacc_w01_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">;
+def mqacc_w11_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">;
+def mqracc_w00_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">;
+def mqracc_w01_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">;
+def mqracc_w11_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">;
+
// Packed Sign and Zero Extend (32-bit)
def psext_b_i16x2 : RISCVBuiltin<"_Vector<2, short>(_Vector<2, short>)">;
def pzext_b_u16x2 : RISCVBuiltin<"_Vector<2, unsigned short>(_Vector<2, unsigned short>)">;
diff --git a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
index 88fedecbee504..91f451e34c2c9 100644
--- a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
@@ -1626,6 +1626,88 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID,
break;
}
+ // Packed "Q-format" Multiply Parts Accumulate
+ case RISCV::BI__builtin_riscv_mqacc_h00_i32:
+ case RISCV::BI__builtin_riscv_mqacc_h01_i32:
+ case RISCV::BI__builtin_riscv_mqacc_h11_i32:
+ case RISCV::BI__builtin_riscv_mqracc_h00_i32:
+ case RISCV::BI__builtin_riscv_mqracc_h01_i32:
+ case RISCV::BI__builtin_riscv_mqracc_h11_i32:
+ case RISCV::BI__builtin_riscv_pmqacc_h00_i32x2:
+ case RISCV::BI__builtin_riscv_pmqacc_h01_i32x2:
+ case RISCV::BI__builtin_riscv_pmqacc_h11_i32x2:
+ case RISCV::BI__builtin_riscv_pmqracc_h00_i32x2:
+ case RISCV::BI__builtin_riscv_pmqracc_h01_i32x2:
+ case RISCV::BI__builtin_riscv_pmqracc_h11_i32x2:
+ case RISCV::BI__builtin_riscv_mqacc_w00_i64:
+ case RISCV::BI__builtin_riscv_mqacc_w01_i64:
+ case RISCV::BI__builtin_riscv_mqacc_w11_i64:
+ case RISCV::BI__builtin_riscv_mqracc_w00_i64:
+ case RISCV::BI__builtin_riscv_mqracc_w01_i64:
+ case RISCV::BI__builtin_riscv_mqracc_w11_i64: {
+ switch (BuiltinID) {
+ default:
+ llvm_unreachable("unexpected builtin ID");
+ case RISCV::BI__builtin_riscv_mqacc_h00_i32:
+ ID = Intrinsic::riscv_mqacc_h00;
+ break;
+ case RISCV::BI__builtin_riscv_mqacc_h01_i32:
+ ID = Intrinsic::riscv_mqacc_h01;
+ break;
+ case RISCV::BI__builtin_riscv_mqacc_h11_i32:
+ ID = Intrinsic::riscv_mqacc_h11;
+ break;
+ case RISCV::BI__builtin_riscv_mqracc_h00_i32:
+ ID = Intrinsic::riscv_mqracc_h00;
+ break;
+ case RISCV::BI__builtin_riscv_mqracc_h01_i32:
+ ID = Intrinsic::riscv_mqracc_h01;
+ break;
+ case RISCV::BI__builtin_riscv_mqracc_h11_i32:
+ ID = Intrinsic::riscv_mqracc_h11;
+ break;
+ case RISCV::BI__builtin_riscv_pmqacc_h00_i32x2:
+ ID = Intrinsic::riscv_pmqacc_h00;
+ break;
+ case RISCV::BI__builtin_riscv_pmqacc_h01_i32x2:
+ ID = Intrinsic::riscv_pmqacc_h01;
+ break;
+ case RISCV::BI__builtin_riscv_pmqacc_h11_i32x2:
+ ID = Intrinsic::riscv_pmqacc_h11;
+ break;
+ case RISCV::BI__builtin_riscv_pmqracc_h00_i32x2:
+ ID = Intrinsic::riscv_pmqracc_h00;
+ break;
+ case RISCV::BI__builtin_riscv_pmqracc_h01_i32x2:
+ ID = Intrinsic::riscv_pmqracc_h01;
+ break;
+ case RISCV::BI__builtin_riscv_pmqracc_h11_i32x2:
+ ID = Intrinsic::riscv_pmqracc_h11;
+ break;
+ case RISCV::BI__builtin_riscv_mqacc_w00_i64:
+ ID = Intrinsic::riscv_mqacc_w00;
+ break;
+ case RISCV::BI__builtin_riscv_mqacc_w01_i64:
+ ID = Intrinsic::riscv_mqacc_w01;
+ break;
+ case RISCV::BI__builtin_riscv_mqacc_w11_i64:
+ ID = Intrinsic::riscv_mqacc_w11;
+ break;
+ case RISCV::BI__builtin_riscv_mqracc_w00_i64:
+ ID = Intrinsic::riscv_mqracc_w00;
+ break;
+ case RISCV::BI__builtin_riscv_mqracc_w01_i64:
+ ID = Intrinsic::riscv_mqracc_w01;
+ break;
+ case RISCV::BI__builtin_riscv_mqracc_w11_i64:
+ ID = Intrinsic::riscv_mqracc_w11;
+ break;
+ }
+
+ IntrinsicTypes = {ResultType, Ops[1]->getType()};
+ break;
+ }
+
// Zk builtins
// Zknh
diff --git a/clang/lib/Headers/riscv_packed_simd.h b/clang/lib/Headers/riscv_packed_simd.h
index 318900805043c..aaf7112d583f4 100644
--- a/clang/lib/Headers/riscv_packed_simd.h
+++ b/clang/lib/Headers/riscv_packed_simd.h
@@ -285,7 +285,7 @@ typedef uint32_t uint32x2_t __attribute__((__vector_size__(8)));
return builtin(__rs1, __rs2); \
}
-#define __packed_abdsum_acc(name, rty, ty, builtin) \
+#define __packed_ternary_builtin(name, rty, ty, builtin) \
static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(rty __rd, ty __rs1, \
ty __rs2) { \
return builtin(__rd, __rs1, __rs2); \
@@ -815,13 +815,13 @@ __packed_ternary_builtin_mixed(pmhraccsu_i32x2, int32x2_t, int32x2_t, uint32x2_t
/* Packed Absolute Difference Sum (32-bit) */
__packed_abdsum(pabdsumu_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumu_u8x4_u32)
-__packed_abdsum_acc(pabdsumau_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumau_u8x4_u32)
+__packed_ternary_builtin(pabdsumau_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumau_u8x4_u32)
/* Packed Absolute Difference Sum (64-bit) */
__packed_abdsum(pabdsumu_u8x8_u32, uint32_t, uint8x8_t, __builtin_riscv_pabdsumu_u8x8_u32)
__packed_abdsum(pabdsumu_u8x8_u64, uint64_t, uint8x8_t, __builtin_riscv_pabdsumu_u8x8_u64)
-__packed_abdsum_acc(pabdsumau_u8x8_u32, uint32_t, uint8x8_t, __builtin_riscv_pabdsumau_u8x8_u32)
-__packed_abdsum_acc(pabdsumau_u8x8_u64, uint64_t, uint8x8_t, __builtin_riscv_pabdsumau_u8x8_u64)
+__packed_ternary_builtin(pabdsumau_u8x8_u32, uint32_t, uint8x8_t, __builtin_riscv_pabdsumau_u8x8_u32)
+__packed_ternary_builtin(pabdsumau_u8x8_u64, uint64_t, uint8x8_t, __builtin_riscv_pabdsumau_u8x8_u64)
/* Packed Saturating Absolute Value (32-bit) */
__packed_unary_builtin(psabs_i8x4, int8x4_t, __builtin_riscv_psabs_i8x4)
@@ -852,6 +852,28 @@ __packed_binary_builtin(pmulqr_i16x4, int16x4_t, __builtin_riscv_pmulqr_i16x4)
__packed_binary_builtin(pmulq_i32x2, int32x2_t, __builtin_riscv_pmulq_i32x2)
__packed_binary_builtin(pmulqr_i32x2, int32x2_t, __builtin_riscv_pmulqr_i32x2)
+/* Packed "Q-format" Multiply Parts Accumulate (32-bit) */
+__packed_ternary_builtin(mqacc_h00_i32, int, int16x2_t, __builtin_riscv_mqacc_h00_i32)
+__packed_ternary_builtin(mqacc_h01_i32, int, int16x2_t, __builtin_riscv_mqacc_h01_i32)
+__packed_ternary_builtin(mqacc_h11_i32, int, int16x2_t, __builtin_riscv_mqacc_h11_i32)
+__packed_ternary_builtin(mqracc_h00_i32, int, int16x2_t, __builtin_riscv_mqracc_h00_i32)
+__packed_ternary_builtin(mqracc_h01_i32, int, int16x2_t, __builtin_riscv_mqracc_h01_i32)
+__packed_ternary_builtin(mqracc_h11_i32, int, int16x2_t, __builtin_riscv_mqracc_h11_i32)
+
+/* Packed "Q-format" Multiply Parts Accumulate (64-bit) */
+__packed_ternary_builtin(pmqacc_h00_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqacc_h00_i32x2)
+__packed_ternary_builtin(pmqacc_h01_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqacc_h01_i32x2)
+__packed_ternary_builtin(pmqacc_h11_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqacc_h11_i32x2)
+__packed_ternary_builtin(pmqracc_h00_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqracc_h00_i32x2)
+__packed_ternary_builtin(pmqracc_h01_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqracc_h01_i32x2)
+__packed_ternary_builtin(pmqracc_h11_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqracc_h11_i32x2)
+__packed_ternary_builtin(mqacc_w00_i64, int64_t, int32x2_t, __builtin_riscv_mqacc_w00_i64)
+__packed_ternary_builtin(mqacc_w01_i64, int64_t, int32x2_t, __builtin_riscv_mqacc_w01_i64)
+__packed_ternary_builtin(mqacc_w11_i64, int64_t, int32x2_t, __builtin_riscv_mqacc_w11_i64)
+__packed_ternary_builtin(mqracc_w00_i64, int64_t, int32x2_t, __builtin_riscv_mqracc_w00_i64)
+__packed_ternary_builtin(mqracc_w01_i64, int64_t, int32x2_t, __builtin_riscv_mqracc_w01_i64)
+__packed_ternary_builtin(mqracc_w11_i64, int64_t, int32x2_t, __builtin_riscv_mqracc_w11_i64)
+
/* Packed Narrowing Clip Pair (32-bit) */
__packed_binary_builtin_cast(pnclipp_i8x4, int16x2_t, int8x4_t, __builtin_riscv_pnclipp_i8x4)
__packed_binary_builtin_cast(pnclipup_u8x4, uint16x2_t, uint8x4_t, __builtin_riscv_pnclipup_u8x4)
@@ -1012,7 +1034,7 @@ __packed_reinterpret(u32x2_i32x2, int32x2_t, uint32x2_t)
#undef __packed_nziph2
#undef __packed_nziph4
#undef __packed_abdsum
-#undef __packed_abdsum_acc
+#undef __packed_ternary_builtin
#undef __packed_reinterpret
#undef __DEFAULT_FN_ATTRS
diff --git a/clang/test/CodeGen/RISCV/rvp-intrinsics.c b/clang/test/CodeGen/RISCV/rvp-intrinsics.c
index 87bc81ed7a645..74a7cb60a6afa 100644
--- a/clang/test/CodeGen/RISCV/rvp-intrinsics.c
+++ b/clang/test/CodeGen/RISCV/rvp-intrinsics.c
@@ -10105,3 +10105,391 @@ int32x2_t test_pnclipp_i32x2(int64_t rs1, int64_t rs2) {
uint32x2_t test_pnclipup_u32x2(uint64_t rs1, uint64_t rs2) {
return __riscv_pnclipup_u32x2(rs1, rs2);
}
+
+/* Packed "Q-format" Multiply Parts Accumulate (32-bit) */
+
+// RV32-LABEL: define dso_local i32 @test_mqacc_h00_i32(
+// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqacc.h00.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV32-NEXT: ret i32 [[TMP2]]
+//
+// RV64-LABEL: define dso_local signext i32 @test_mqacc_h00_i32(
+// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqacc.h00.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV64-NEXT: ret i32 [[TMP2]]
+//
+
+int32_t test_mqacc_h00_i32(int32_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_mqacc_h00_i32(rd, rs1, rs2);
+}
+// RV32-LABEL: define dso_local i32 @test_mqacc_h01_i32(
+// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqacc.h01.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV32-NEXT: ret i32 [[TMP2]]
+//
+// RV64-LABEL: define dso_local signext i32 @test_mqacc_h01_i32(
+// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqacc.h01.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV64-NEXT: ret i32 [[TMP2]]
+//
+int32_t test_mqacc_h01_i32(int32_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_mqacc_h01_i32(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i32 @test_mqacc_h11_i32(
+// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqacc.h11.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV32-NEXT: ret i32 [[TMP2]]
+//
+// RV64-LABEL: define dso_local signext i32 @test_mqacc_h11_i32(
+// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqacc.h11.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV64-NEXT: ret i32 [[TMP2]]
+//
+int32_t test_mqacc_h11_i32(int32_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_mqacc_h11_i32(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i32 @test_mqracc_h00_i32(
+// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqracc.h00.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV32-NEXT: ret i32 [[TMP2]]
+//
+// RV64-LABEL: define dso_local signext i32 @test_mqracc_h00_i32(
+// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqracc.h00.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV64-NEXT: ret i32 [[TMP2]]
+//
+int32_t test_mqracc_h00_i32(int32_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_mqracc_h00_i32(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i32 @test_mqracc_h01_i32(
+// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqracc.h01.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV32-NEXT: ret i32 [[TMP2]]
+//
+// RV64-LABEL: define dso_local signext i32 @test_mqracc_h01_i32(
+// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqracc.h01.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV64-NEXT: ret i32 [[TMP2]]
+//
+int32_t test_mqracc_h01_i32(int32_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_mqracc_h01_i32(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i32 @test_mqracc_h11_i32(
+// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqracc.h11.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV32-NEXT: ret i32 [[TMP2]]
+//
+// RV64-LABEL: define dso_local signext i32 @test_mqracc_h11_i32(
+// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16>
+// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqracc.h11.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+// RV64-NEXT: ret i32 [[TMP2]]
+//
+int32_t test_mqracc_h11_i32(int32_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_mqracc_h11_i32(rd, rs1, rs2);
+}
+
+/* Packed "Q-format" Multiply Parts Accumulate (64-bit) */
+
+// RV32-LABEL: define dso_local i64 @test_pmqacc_h00_i32x2(
+// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// RV32-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// RV32-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqacc.h00.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+// RV32-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
+// RV32-NEXT: ret i64 [[TMP4]]
+//
+// RV64-LABEL: define dso_local i64 @test_pmqacc_h00_i32x2(
+// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// RV64-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// RV64-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqacc.h00.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+// RV64-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
+// RV64-NEXT: ret i64 [[TMP4]]
+//
+int32x2_t test_pmqacc_h00_i32x2(int32x2_t rd, int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pmqacc_h00_i32x2(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_pmqacc_h01_i32x2(
+// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// RV32-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// RV32-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqacc.h01.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+// RV32-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
+// RV32-NEXT: ret i64 [[TMP4]]
+//
+// RV64-LABEL: define dso_local i64 @test_pmqacc_h01_i32x2(
+// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// RV64-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// RV64-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqacc.h01.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+// RV64-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
+// RV64-NEXT: ret i64 [[TMP4]]
+//
+int32x2_t test_pmqacc_h01_i32x2(int32x2_t rd, int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pmqacc_h01_i32x2(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_pmqacc_h11_i32x2(
+// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// RV32-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// RV32-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqacc.h11.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+// RV32-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
+// RV32-NEXT: ret i64 [[TMP4]]
+//
+// RV64-LABEL: define dso_local i64 @test_pmqacc_h11_i32x2(
+// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// RV64-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// RV64-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqacc.h11.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+// RV64-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
+// RV64-NEXT: ret i64 [[TMP4]]
+//
+int32x2_t test_pmqacc_h11_i32x2(int32x2_t rd, int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pmqacc_h11_i32x2(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_pmqracc_h00_i32x2(
+// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// RV32-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// RV32-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqracc.h00.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+// RV32-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
+// RV32-NEXT: ret i64 [[TMP4]]
+//
+// RV64-LABEL: define dso_local i64 @test_pmqracc_h00_i32x2(
+// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// RV64-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// RV64-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqracc.h00.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+// RV64-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
+// RV64-NEXT: ret i64 [[TMP4]]
+//
+int32x2_t test_pmqracc_h00_i32x2(int32x2_t rd, int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pmqracc_h00_i32x2(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_pmqracc_h01_i32x2(
+// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// RV32-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// RV32-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqracc.h01.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+// RV32-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
+// RV32-NEXT: ret i64 [[TMP4]]
+//
+// RV64-LABEL: define dso_local i64 @test_pmqracc_h01_i32x2(
+// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// RV64-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// RV64-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqracc.h01.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+// RV64-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
+// RV64-NEXT: ret i64 [[TMP4]]
+//
+int32x2_t test_pmqracc_h01_i32x2(int32x2_t rd, int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pmqracc_h01_i32x2(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_pmqracc_h11_i32x2(
+// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// RV32-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// RV32-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqracc.h11.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+// RV32-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
+// RV32-NEXT: ret i64 [[TMP4]]
+//
+// RV64-LABEL: define dso_local i64 @test_pmqracc_h11_i32x2(
+// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16>
+// RV64-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16>
+// RV64-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqracc.h11.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]])
+// RV64-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64
+// RV64-NEXT: ret i64 [[TMP4]]
+//
+int32x2_t test_pmqracc_h11_i32x2(int32x2_t rd, int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pmqracc_h11_i32x2(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_mqacc_w00_i64(
+// RV32-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32>
+// RV32-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqacc.w00.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]])
+// RV32-NEXT: ret i64 [[TMP2]]
+//
+// RV64-LABEL: define dso_local i64 @test_mqacc_w00_i64(
+// RV64-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32>
+// RV64-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqacc.w00.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]])
+// RV64-NEXT: ret i64 [[TMP2]]
+//
+int64_t test_mqacc_w00_i64(int64_t rd, int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_mqacc_w00_i64(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_mqacc_w01_i64(
+// RV32-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32>
+// RV32-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqacc.w01.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]])
+// RV32-NEXT: ret i64 [[TMP2]]
+//
+// RV64-LABEL: define dso_local i64 @test_mqacc_w01_i64(
+// RV64-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32>
+// RV64-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqacc.w01.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]])
+// RV64-NEXT: ret i64 [[TMP2]]
+//
+int64_t test_mqacc_w01_i64(int64_t rd, int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_mqacc_w01_i64(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_mqacc_w11_i64(
+// RV32-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32>
+// RV32-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqacc.w11.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]])
+// RV32-NEXT: ret i64 [[TMP2]]
+//
+// RV64-LABEL: define dso_local i64 @test_mqacc_w11_i64(
+// RV64-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32>
+// RV64-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqacc.w11.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]])
+// RV64-NEXT: ret i64 [[TMP2]]
+//
+int64_t test_mqacc_w11_i64(int64_t rd, int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_mqacc_w11_i64(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_mqracc_w00_i64(
+// RV32-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32>
+// RV32-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqracc.w00.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]])
+// RV32-NEXT: ret i64 [[TMP2]]
+//
+// RV64-LABEL: define dso_local i64 @test_mqracc_w00_i64(
+// RV64-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32>
+// RV64-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqracc.w00.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]])
+// RV64-NEXT: ret i64 [[TMP2]]
+//
+int64_t test_mqracc_w00_i64(int64_t rd, int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_mqracc_w00_i64(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_mqracc_w01_i64(
+// RV32-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32>
+// RV32-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqracc.w01.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]])
+// RV32-NEXT: ret i64 [[TMP2]]
+//
+// RV64-LABEL: define dso_local i64 @test_mqracc_w01_i64(
+// RV64-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32>
+// RV64-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqracc.w01.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]])
+// RV64-NEXT: ret i64 [[TMP2]]
+//
+int64_t test_mqracc_w01_i64(int64_t rd, int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_mqracc_w01_i64(rd, rs1, rs2);
+}
+
+// RV32-LABEL: define dso_local i64 @test_mqracc_w11_i64(
+// RV32-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV32-NEXT: [[ENTRY:.*:]]
+// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32>
+// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32>
+// RV32-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqracc.w11.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]])
+// RV32-NEXT: ret i64 [[TMP2]]
+//
+// RV64-LABEL: define dso_local i64 @test_mqracc_w11_i64(
+// RV64-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] {
+// RV64-NEXT: [[ENTRY:.*:]]
+// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32>
+// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32>
+// RV64-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqracc.w11.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]])
+// RV64-NEXT: ret i64 [[TMP2]]
+//
+int64_t test_mqracc_w11_i64(int64_t rd, int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_mqracc_w11_i64(rd, rs1, rs2);
+}
diff --git a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
index 75dcb796b66cd..28e3a8911739d 100644
--- a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
+++ b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
@@ -3406,3 +3406,129 @@ int32x2_t test_pnclipp_i32x2(int64_t a, int64_t b) {
uint32x2_t test_pnclipup_u32x2(uint64_t a, uint64_t b) {
return __riscv_pnclipup_u32x2(a, b);
}
+
+// CHECK-LABEL: test_mqacc_h00_i32:
+// RV32: mqacc.h00
+// RV64: pmqacc.w.h00
+int32_t test_mqacc_h00_i32(int32_t rd, int16x2_t a, int16x2_t b) {
+ return __riscv_mqacc_h00_i32(rd, a, b);
+}
+
+// CHECK-LABEL: test_mqacc_h01_i32:
+// RV32: mqacc.h01
+// RV64: pmqacc.w.h01
+int32_t test_mqacc_h01_i32(int32_t rd, int16x2_t a, int16x2_t b) {
+ return __riscv_mqacc_h01_i32(rd, a, b);
+}
+
+// CHECK-LABEL: test_mqacc_h11_i32:
+// RV32: mqacc.h11
+// RV64: pmqacc.w.h11
+int32_t test_mqacc_h11_i32(int32_t rd, int16x2_t a, int16x2_t b) {
+ return __riscv_mqacc_h11_i32(rd, a, b);
+}
+
+// CHECK-LABEL: test_mqracc_h00_i32:
+// RV32: mqracc.h00
+// RV64: pmqracc.w.h00
+int32_t test_mqracc_h00_i32(int32_t rd, int16x2_t a, int16x2_t b) {
+ return __riscv_mqracc_h00_i32(rd, a, b);
+}
+
+// CHECK-LABEL: test_mqracc_h01_i32:
+// RV32: mqracc.h01
+// RV64: pmqracc.w.h01
+int32_t test_mqracc_h01_i32(int32_t rd, int16x2_t a, int16x2_t b) {
+ return __riscv_mqracc_h01_i32(rd, a, b);
+}
+
+// CHECK-LABEL: test_mqracc_h11_i32:
+// RV32: mqracc.h11
+// RV64: pmqracc.w.h11
+int32_t test_mqracc_h11_i32(int32_t rd, int16x2_t a, int16x2_t b) {
+ return __riscv_mqracc_h11_i32(rd, a, b);
+}
+
+// CHECK-LABEL: test_pmqacc_h00_i32x2:
+// RV32-COUNT-2: mqacc.h00
+// RV64: pmqacc.w.h00
+int32x2_t test_pmqacc_h00_i32x2(int32x2_t rd, int16x4_t a, int16x4_t b) {
+ return __riscv_pmqacc_h00_i32x2(rd, a, b);
+}
+
+// CHECK-LABEL: test_pmqacc_h01_i32x2:
+// RV32-COUNT-2: mqacc.h01
+// RV64: pmqacc.w.h01
+int32x2_t test_pmqacc_h01_i32x2(int32x2_t rd, int16x4_t a, int16x4_t b) {
+ return __riscv_pmqacc_h01_i32x2(rd, a, b);
+}
+
+// CHECK-LABEL: test_pmqacc_h11_i32x2:
+// RV32-COUNT-2: mqacc.h11
+// RV64: pmqacc.w.h11
+int32x2_t test_pmqacc_h11_i32x2(int32x2_t rd, int16x4_t a, int16x4_t b) {
+ return __riscv_pmqacc_h11_i32x2(rd, a, b);
+}
+
+// CHECK-LABEL: test_pmqracc_h00_i32x2:
+// RV32-COUNT-2: mqracc.h00
+// RV64: pmqracc.w.h00
+int32x2_t test_pmqracc_h00_i32x2(int32x2_t rd, int16x4_t a, int16x4_t b) {
+ return __riscv_pmqracc_h00_i32x2(rd, a, b);
+}
+
+// CHECK-LABEL: test_pmqracc_h01_i32x2:
+// RV32-COUNT-2: mqracc.h01
+// RV64: pmqracc.w.h01
+int32x2_t test_pmqracc_h01_i32x2(int32x2_t rd, int16x4_t a, int16x4_t b) {
+ return __riscv_pmqracc_h01_i32x2(rd, a, b);
+}
+
+// CHECK-LABEL: test_pmqracc_h11_i32x2:
+// RV32-COUNT-2: mqracc.h11
+// RV64: pmqracc.w.h11
+int32x2_t test_pmqracc_h11_i32x2(int32x2_t rd, int16x4_t a, int16x4_t b) {
+ return __riscv_pmqracc_h11_i32x2(rd, a, b);
+}
+
+// CHECK-LABEL: test_mqacc_w00_i64:
+// RV32: mqwacc
+// RV64: mqacc.w00
+int64_t test_mqacc_w00_i64(int64_t rd, int32x2_t a, int32x2_t b) {
+ return __riscv_mqacc_w00_i64(rd, a, b);
+}
+
+// CHECK-LABEL: test_mqacc_w01_i64:
+// RV32: mqwacc
+// RV64: mqacc.w01
+int64_t test_mqacc_w01_i64(int64_t rd, int32x2_t a, int32x2_t b) {
+ return __riscv_mqacc_w01_i64(rd, a, b);
+}
+
+// CHECK-LABEL: test_mqacc_w11_i64:
+// RV32: mqwacc
+// RV64: mqacc.w11
+int64_t test_mqacc_w11_i64(int64_t rd, int32x2_t a, int32x2_t b) {
+ return __riscv_mqacc_w11_i64(rd, a, b);
+}
+
+// CHECK-LABEL: test_mqracc_w00_i64:
+// RV32: mqrwacc
+// RV64: mqracc.w00
+int64_t test_mqracc_w00_i64(int64_t rd, int32x2_t a, int32x2_t b) {
+ return __riscv_mqracc_w00_i64(rd, a, b);
+}
+
+// CHECK-LABEL: test_mqracc_w01_i64:
+// RV32: mqrwacc
+// RV64: mqracc.w01
+int64_t test_mqracc_w01_i64(int64_t rd, int32x2_t a, int32x2_t b) {
+ return __riscv_mqracc_w01_i64(rd, a, b);
+}
+
+// CHECK-LABEL: test_mqracc_w11_i64:
+// RV32: mqrwacc
+// RV64: mqracc.w11
+int64_t test_mqracc_w11_i64(int64_t rd, int32x2_t a, int32x2_t b) {
+ return __riscv_mqracc_w11_i64(rd, a, b);
+}
diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td
index 8fa0ee78ca2a5..9c810ecc2a624 100644
--- a/llvm/include/llvm/IR/IntrinsicsRISCV.td
+++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td
@@ -2150,6 +2150,31 @@ class RVPBinaryIntrinsic
[IntrNoMem, IntrSpeculatable]>;
def int_riscv_pnclipp : RVPNarrowingClipIntrinsic;
def int_riscv_pnclipup : RVPNarrowingClipIntrinsic;
+
+ // Packed "Q-format" Multiply Parts Accumulate
+ class RVPQFormatAccIntrinsic
+ : DefaultAttrsIntrinsic<[llvm_anyint_ty],
+ [LLVMMatchType<0>, llvm_any_ty,
+ LLVMMatchType<1>],
+ [IntrNoMem, IntrSpeculatable]>;
+ def int_riscv_mqacc_h00 : RVPQFormatAccIntrinsic;
+ def int_riscv_mqacc_h01 : RVPQFormatAccIntrinsic;
+ def int_riscv_mqacc_h11 : RVPQFormatAccIntrinsic;
+ def int_riscv_mqracc_h00 : RVPQFormatAccIntrinsic;
+ def int_riscv_mqracc_h01 : RVPQFormatAccIntrinsic;
+ def int_riscv_mqracc_h11 : RVPQFormatAccIntrinsic;
+ def int_riscv_pmqacc_h00 : RVPQFormatAccIntrinsic;
+ def int_riscv_pmqacc_h01 : RVPQFormatAccIntrinsic;
+ def int_riscv_pmqacc_h11 : RVPQFormatAccIntrinsic;
+ def int_riscv_pmqracc_h00 : RVPQFormatAccIntrinsic;
+ def int_riscv_pmqracc_h01 : RVPQFormatAccIntrinsic;
+ def int_riscv_pmqracc_h11 : RVPQFormatAccIntrinsic;
+ def int_riscv_mqacc_w00 : RVPQFormatAccIntrinsic;
+ def int_riscv_mqacc_w01 : RVPQFormatAccIntrinsic;
+ def int_riscv_mqacc_w11 : RVPQFormatAccIntrinsic;
+ def int_riscv_mqracc_w00 : RVPQFormatAccIntrinsic;
+ def int_riscv_mqracc_w01 : RVPQFormatAccIntrinsic;
+ def int_riscv_mqracc_w11 : RVPQFormatAccIntrinsic;
} // TargetPrefix = "riscv"
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 017c9aac37230..29d133b5adba9 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -2073,6 +2073,22 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
CurDAG->RemoveDeadNode(Node);
return;
}
+ case RISCVISD::MQWACC:
+ case RISCVISD::MQRWACC: {
+ assert(!Subtarget->is64Bit() && Subtarget->hasStdExtP() &&
+ "Unexpected opcode");
+
+ SDValue Op0 = buildGPRPair(CurDAG, DL, MVT::Untyped, Node->getOperand(0),
+ Node->getOperand(1));
+ unsigned Opc = Opcode == RISCVISD::MQRWACC ? RISCV::MQRWACC : RISCV::MQWACC;
+ MachineSDNode *New = CurDAG->getMachineNode(
+ Opc, DL, MVT::Untyped, Op0, Node->getOperand(2), Node->getOperand(3));
+ auto [Lo, Hi] = extractGPRPair(CurDAG, DL, SDValue(New, 0));
+ ReplaceUses(SDValue(Node, 0), Lo);
+ ReplaceUses(SDValue(Node, 1), Hi);
+ CurDAG->RemoveDeadNode(Node);
+ return;
+ }
case RISCVISD::ADDD:
// Try to match WMACC pattern: ADDD where one operand pair comes from a
// widening multiply.
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 194aad22775cf..63d3522f88e47 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -12181,8 +12181,8 @@ static void processVCIXOperands(SDValue OrigOp,
// LMUL * VLEN should be greater than or equal to EGS * SEW
static inline bool isValidEGW(int EGS, EVT VT,
const RISCVSubtarget &Subtarget) {
- return (Subtarget.getRealMinVLen() *
- VT.getSizeInBits().getKnownMinValue()) / RISCV::RVVBitsPerBlock >=
+ return (Subtarget.getRealMinVLen() * VT.getSizeInBits().getKnownMinValue()) /
+ RISCV::RVVBitsPerBlock >=
EGS * VT.getScalarSizeInBits();
}
@@ -12254,6 +12254,94 @@ static unsigned getRVPMulHighAccumulateOpcode(unsigned IntNo) {
}
}
+static unsigned getRVPQFormatAccScalarOpcode(Intrinsic::ID IntNo) {
+ switch (IntNo) {
+ default:
+ llvm_unreachable("Unexpected RISC-V packed Q-format accumulate intrinsic");
+ case Intrinsic::riscv_mqacc_h00:
+ return RISCVISD::MQACC_H00;
+ case Intrinsic::riscv_mqacc_h01:
+ return RISCVISD::MQACC_H01;
+ case Intrinsic::riscv_mqacc_h11:
+ return RISCVISD::MQACC_H11;
+ case Intrinsic::riscv_mqracc_h00:
+ return RISCVISD::MQRACC_H00;
+ case Intrinsic::riscv_mqracc_h01:
+ return RISCVISD::MQRACC_H01;
+ case Intrinsic::riscv_mqracc_h11:
+ return RISCVISD::MQRACC_H11;
+ case Intrinsic::riscv_pmqacc_h00:
+ return RISCVISD::MQACC_H00;
+ case Intrinsic::riscv_pmqacc_h01:
+ return RISCVISD::MQACC_H01;
+ case Intrinsic::riscv_pmqacc_h11:
+ return RISCVISD::MQACC_H11;
+ case Intrinsic::riscv_pmqracc_h00:
+ return RISCVISD::MQRACC_H00;
+ case Intrinsic::riscv_pmqracc_h01:
+ return RISCVISD::MQRACC_H01;
+ case Intrinsic::riscv_pmqracc_h11:
+ return RISCVISD::MQRACC_H11;
+ }
+}
+
+static unsigned getRVPQFormatAccOpcode(Intrinsic::ID IntNo) {
+ switch (IntNo) {
+ default:
+ llvm_unreachable("Unexpected RISC-V packed Q-format accumulate intrinsic");
+ case Intrinsic::riscv_pmqacc_h00:
+ return RISCVISD::PMQACC_W_H00;
+ case Intrinsic::riscv_pmqacc_h01:
+ return RISCVISD::PMQACC_W_H01;
+ case Intrinsic::riscv_pmqacc_h11:
+ return RISCVISD::PMQACC_W_H11;
+ case Intrinsic::riscv_pmqracc_h00:
+ return RISCVISD::PMQRACC_W_H00;
+ case Intrinsic::riscv_pmqracc_h01:
+ return RISCVISD::PMQRACC_W_H01;
+ case Intrinsic::riscv_pmqracc_h11:
+ return RISCVISD::PMQRACC_W_H11;
+ }
+}
+
+static unsigned getRVPQFormatAccWordOpcode(Intrinsic::ID IntNo) {
+ switch (IntNo) {
+ default:
+ llvm_unreachable("Unexpected RISC-V packed Q-format accumulate intrinsic");
+ case Intrinsic::riscv_mqacc_w00:
+ return RISCVISD::MQACC_W00;
+ case Intrinsic::riscv_mqacc_w01:
+ return RISCVISD::MQACC_W01;
+ case Intrinsic::riscv_mqacc_w11:
+ return RISCVISD::MQACC_W11;
+ case Intrinsic::riscv_mqracc_w00:
+ return RISCVISD::MQRACC_W00;
+ case Intrinsic::riscv_mqracc_w01:
+ return RISCVISD::MQRACC_W01;
+ case Intrinsic::riscv_mqracc_w11:
+ return RISCVISD::MQRACC_W11;
+ }
+}
+
+static unsigned getRVPQFormatAccWideOpcode(Intrinsic::ID IntNo) {
+ switch (IntNo) {
+ default:
+ llvm_unreachable("Unexpected RISC-V packed Q-format accumulate intrinsic");
+ case Intrinsic::riscv_mqacc_h00:
+ return RISCVISD::PMQACC_W_H00_W;
+ case Intrinsic::riscv_mqacc_h01:
+ return RISCVISD::PMQACC_W_H01_W;
+ case Intrinsic::riscv_mqacc_h11:
+ return RISCVISD::PMQACC_W_H11_W;
+ case Intrinsic::riscv_mqracc_h00:
+ return RISCVISD::PMQRACC_W_H00_W;
+ case Intrinsic::riscv_mqracc_h01:
+ return RISCVISD::PMQRACC_W_H01_W;
+ case Intrinsic::riscv_mqracc_h11:
+ return RISCVISD::PMQRACC_W_H11_W;
+ }
+}
+
SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
SelectionDAG &DAG) const {
unsigned IntNo = Op.getConstantOperandVal(0);
@@ -12347,12 +12435,24 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
switch (IntNo) {
case Intrinsic::riscv_orc_b: Opc = RISCVISD::ORC_B; break;
case Intrinsic::riscv_brev8: Opc = RISCVISD::BREV8; break;
- case Intrinsic::riscv_sha256sig0: Opc = RISCVISD::SHA256SIG0; break;
- case Intrinsic::riscv_sha256sig1: Opc = RISCVISD::SHA256SIG1; break;
- case Intrinsic::riscv_sha256sum0: Opc = RISCVISD::SHA256SUM0; break;
- case Intrinsic::riscv_sha256sum1: Opc = RISCVISD::SHA256SUM1; break;
- case Intrinsic::riscv_sm3p0: Opc = RISCVISD::SM3P0; break;
- case Intrinsic::riscv_sm3p1: Opc = RISCVISD::SM3P1; break;
+ case Intrinsic::riscv_sha256sig0:
+ Opc = RISCVISD::SHA256SIG0;
+ break;
+ case Intrinsic::riscv_sha256sig1:
+ Opc = RISCVISD::SHA256SIG1;
+ break;
+ case Intrinsic::riscv_sha256sum0:
+ Opc = RISCVISD::SHA256SUM0;
+ break;
+ case Intrinsic::riscv_sha256sum1:
+ Opc = RISCVISD::SHA256SUM1;
+ break;
+ case Intrinsic::riscv_sm3p0:
+ Opc = RISCVISD::SM3P0;
+ break;
+ case Intrinsic::riscv_sm3p1:
+ Opc = RISCVISD::SM3P1;
+ break;
}
return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1));
@@ -12467,6 +12567,72 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
llvm_unreachable("unexpected VT for pnclipp/pnclipup on RV32");
}
+ case Intrinsic::riscv_mqacc_h00:
+ case Intrinsic::riscv_mqacc_h01:
+ case Intrinsic::riscv_mqacc_h11:
+ case Intrinsic::riscv_mqracc_h00:
+ case Intrinsic::riscv_mqracc_h01:
+ case Intrinsic::riscv_mqracc_h11:
+ case Intrinsic::riscv_pmqacc_h00:
+ case Intrinsic::riscv_pmqacc_h01:
+ case Intrinsic::riscv_pmqacc_h11:
+ case Intrinsic::riscv_pmqracc_h00:
+ case Intrinsic::riscv_pmqracc_h01:
+ case Intrinsic::riscv_pmqracc_h11:
+ case Intrinsic::riscv_mqacc_w00:
+ case Intrinsic::riscv_mqacc_w01:
+ case Intrinsic::riscv_mqacc_w11:
+ case Intrinsic::riscv_mqracc_w00:
+ case Intrinsic::riscv_mqracc_w01:
+ case Intrinsic::riscv_mqracc_w11: {
+ EVT VT = Op.getValueType();
+ SDValue Rd = Op.getOperand(1);
+ SDValue Rs1 = Op.getOperand(2);
+ SDValue Rs2 = Op.getOperand(3);
+ MVT XLenVT = Subtarget.getXLenVT();
+
+ bool IsScalarHalfword = IntNo == Intrinsic::riscv_mqacc_h00 ||
+ IntNo == Intrinsic::riscv_mqacc_h01 ||
+ IntNo == Intrinsic::riscv_mqacc_h11 ||
+ IntNo == Intrinsic::riscv_mqracc_h00 ||
+ IntNo == Intrinsic::riscv_mqracc_h01 ||
+ IntNo == Intrinsic::riscv_mqracc_h11;
+ if (Subtarget.is64Bit() && IsScalarHalfword)
+ return SDValue();
+
+ if (VT == MVT::v2i32 && Rs1.getSimpleValueType() == MVT::v4i16) {
+ if (Subtarget.is64Bit()) {
+ unsigned Opc = getRVPQFormatAccOpcode(IntNo);
+ return DAG.getNode(Opc, DL, VT, Rd, Rs1, Rs2);
+ }
+
+ auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
+ auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
+ SDValue RdLo = DAG.getExtractVectorElt(DL, XLenVT, Rd, 0);
+ SDValue RdHi = DAG.getExtractVectorElt(DL, XLenVT, Rd, 1);
+ unsigned ScalarOpc = getRVPQFormatAccScalarOpcode(IntNo);
+ SDValue Lo = DAG.getNode(ScalarOpc, DL, XLenVT, RdLo,
+ DAG.getBitcast(XLenVT, Rs1Lo),
+ DAG.getBitcast(XLenVT, Rs2Lo));
+ SDValue Hi = DAG.getNode(ScalarOpc, DL, XLenVT, RdHi,
+ DAG.getBitcast(XLenVT, Rs1Hi),
+ DAG.getBitcast(XLenVT, Rs2Hi));
+ return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Lo, Hi);
+ }
+
+ if (VT == MVT::i32 && !Subtarget.is64Bit()) {
+ unsigned Opc = getRVPQFormatAccScalarOpcode(IntNo);
+ return DAG.getNode(Opc, DL, XLenVT, Rd, DAG.getBitcast(XLenVT, Rs1),
+ DAG.getBitcast(XLenVT, Rs2));
+ }
+
+ if (VT == MVT::i64 && Subtarget.is64Bit()) {
+ unsigned Opc = getRVPQFormatAccWordOpcode(IntNo);
+ return DAG.getNode(Opc, DL, XLenVT, Rd, Rs1, Rs2);
+ }
+
+ return SDValue();
+ }
case Intrinsic::riscv_pmulq:
case Intrinsic::riscv_pmulqr: {
unsigned Opc;
@@ -16963,6 +17129,63 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
return;
}
+ case Intrinsic::riscv_mqacc_h00:
+ case Intrinsic::riscv_mqacc_h01:
+ case Intrinsic::riscv_mqacc_h11:
+ case Intrinsic::riscv_mqracc_h00:
+ case Intrinsic::riscv_mqracc_h01:
+ case Intrinsic::riscv_mqracc_h11: {
+ EVT VT = N->getValueType(0);
+ if (!Subtarget.is64Bit() || VT != MVT::i32)
+ return;
+ MVT XLenVT = Subtarget.getXLenVT();
+ // Widen a v2i16 source to XLenVT: bitcast to the 32-bit scalar it shares
+ // bits with, then zero-extend into the upper half of the GPR.
+ auto WidenSrc = [&](SDValue V) {
+ if (V.getValueType() == XLenVT)
+ return V;
+ assert(V.getValueType() == MVT::v2i16);
+ return DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT,
+ DAG.getBitcast(MVT::i32, V));
+ };
+ SDValue Rd = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(1));
+ SDValue Rs1 = WidenSrc(N->getOperand(2));
+ SDValue Rs2 = WidenSrc(N->getOperand(3));
+ unsigned Opc = getRVPQFormatAccWideOpcode(IntNo);
+ SDValue Res = DAG.getNode(Opc, DL, XLenVT, Rd, Rs1, Rs2);
+ Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
+ return;
+ }
+ case Intrinsic::riscv_mqacc_w00:
+ case Intrinsic::riscv_mqacc_w01:
+ case Intrinsic::riscv_mqacc_w11:
+ case Intrinsic::riscv_mqracc_w00:
+ case Intrinsic::riscv_mqracc_w01:
+ case Intrinsic::riscv_mqracc_w11: {
+ if (Subtarget.is64Bit() || N->getValueType(0) != MVT::i64)
+ return;
+ bool IsRound = IntNo == Intrinsic::riscv_mqracc_w00 ||
+ IntNo == Intrinsic::riscv_mqracc_w01 ||
+ IntNo == Intrinsic::riscv_mqracc_w11;
+ bool Hi1 = IntNo == Intrinsic::riscv_mqacc_w11 ||
+ IntNo == Intrinsic::riscv_mqracc_w11;
+ bool Hi2 = IntNo == Intrinsic::riscv_mqacc_w01 ||
+ IntNo == Intrinsic::riscv_mqacc_w11 ||
+ IntNo == Intrinsic::riscv_mqracc_w01 ||
+ IntNo == Intrinsic::riscv_mqracc_w11;
+ MVT XLenVT = Subtarget.getXLenVT();
+ SDValue Rs1 = N->getOperand(2);
+ SDValue Rs2 = N->getOperand(3);
+ SDValue A = DAG.getExtractVectorElt(DL, XLenVT, Rs1, Hi1 ? 1 : 0);
+ SDValue B = DAG.getExtractVectorElt(DL, XLenVT, Rs2, Hi2 ? 1 : 0);
+ auto [RdLo, RdHi] = DAG.SplitScalar(N->getOperand(1), DL, XLenVT, XLenVT);
+ unsigned Opc = IsRound ? RISCVISD::MQRWACC : RISCVISD::MQWACC;
+ SDVTList VTs = DAG.getVTList(XLenVT, XLenVT);
+ SDValue Acc = DAG.getNode(Opc, DL, VTs, {RdLo, RdHi, A, B});
+ Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64,
+ Acc.getValue(0), Acc.getValue(1)));
+ return;
+ }
case Intrinsic::riscv_orc_b:
case Intrinsic::riscv_brev8:
case Intrinsic::riscv_sha256sig0:
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
index 9b72141ef9bcd..799949da7ce2b 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
@@ -1966,6 +1966,50 @@ def SDT_RISCVPackedNarrowingClipW
def riscv_pnclipp_w : RVSDNode<"PNCLIPP_W", SDT_RISCVPackedNarrowingClipW>;
def riscv_pnclipup_w : RVSDNode<"PNCLIPUP_W", SDT_RISCVPackedNarrowingClipW>;
+// Packed "Q-format" multiply parts accumulate.
+def SDT_RISCVQFormatAccS
+ : SDTypeProfile<1, 3, [SDTCisVT<0, XLenVT>, SDTCisVT<1, XLenVT>,
+ SDTCisVT<2, XLenVT>,
+ SDTCisSameAs<2, 3>]>;
+def SDT_RISCVQFormatAccP
+ : SDTypeProfile<1, 3, [SDTCisVT<0, v2i32>, SDTCisVT<1, v2i32>,
+ SDTCisVT<2, v4i16>, SDTCisSameAs<2, 3>]>;
+def SDT_RISCVQFormatAccW
+ : SDTypeProfile<1, 3, [SDTCisVT<0, XLenVT>, SDTCisVT<1, XLenVT>,
+ SDTCisVT<2, v2i32>,
+ SDTCisSameAs<2, 3>]>;
+
+def riscv_mqacc_h00 : RVSDNode<"MQACC_H00", SDT_RISCVQFormatAccS>;
+def riscv_mqacc_h01 : RVSDNode<"MQACC_H01", SDT_RISCVQFormatAccS>;
+def riscv_mqacc_h11 : RVSDNode<"MQACC_H11", SDT_RISCVQFormatAccS>;
+def riscv_mqracc_h00 : RVSDNode<"MQRACC_H00", SDT_RISCVQFormatAccS>;
+def riscv_mqracc_h01 : RVSDNode<"MQRACC_H01", SDT_RISCVQFormatAccS>;
+def riscv_mqracc_h11 : RVSDNode<"MQRACC_H11", SDT_RISCVQFormatAccS>;
+
+def riscv_pmqacc_h00 : RVSDNode<"PMQACC_W_H00", SDT_RISCVQFormatAccP>;
+def riscv_pmqacc_h01 : RVSDNode<"PMQACC_W_H01", SDT_RISCVQFormatAccP>;
+def riscv_pmqacc_h11 : RVSDNode<"PMQACC_W_H11", SDT_RISCVQFormatAccP>;
+def riscv_pmqracc_h00 : RVSDNode<"PMQRACC_W_H00", SDT_RISCVQFormatAccP>;
+def riscv_pmqracc_h01 : RVSDNode<"PMQRACC_W_H01", SDT_RISCVQFormatAccP>;
+def riscv_pmqracc_h11 : RVSDNode<"PMQRACC_W_H11", SDT_RISCVQFormatAccP>;
+
+def riscv_pmqacc_w_h00 : RVSDNode<"PMQACC_W_H00_W", SDT_RISCVQFormatAccS>;
+def riscv_pmqacc_w_h01 : RVSDNode<"PMQACC_W_H01_W", SDT_RISCVQFormatAccS>;
+def riscv_pmqacc_w_h11 : RVSDNode<"PMQACC_W_H11_W", SDT_RISCVQFormatAccS>;
+def riscv_pmqracc_w_h00 : RVSDNode<"PMQRACC_W_H00_W", SDT_RISCVQFormatAccS>;
+def riscv_pmqracc_w_h01 : RVSDNode<"PMQRACC_W_H01_W", SDT_RISCVQFormatAccS>;
+def riscv_pmqracc_w_h11 : RVSDNode<"PMQRACC_W_H11_W", SDT_RISCVQFormatAccS>;
+
+def riscv_mqacc_w00 : RVSDNode<"MQACC_W00", SDT_RISCVQFormatAccW>;
+def riscv_mqacc_w01 : RVSDNode<"MQACC_W01", SDT_RISCVQFormatAccW>;
+def riscv_mqacc_w11 : RVSDNode<"MQACC_W11", SDT_RISCVQFormatAccW>;
+def riscv_mqracc_w00 : RVSDNode<"MQRACC_W00", SDT_RISCVQFormatAccW>;
+def riscv_mqracc_w01 : RVSDNode<"MQRACC_W01", SDT_RISCVQFormatAccW>;
+def riscv_mqracc_w11 : RVSDNode<"MQRACC_W11", SDT_RISCVQFormatAccW>;
+
+def riscv_mqwacc : RVSDNode<"MQWACC", SDT_RISCVWideningAddSubAccumulate>;
+def riscv_mqrwacc : RVSDNode<"MQRWACC", SDT_RISCVWideningAddSubAccumulate>;
+
// The immediate for these is the number of trailing ones in the max value.
def riscv_sati : RVSDNode<"SATI", SDTIntBinOp>;
def riscv_usati : RVSDNode<"USATI", SDTIntBinOp>;
@@ -2729,6 +2773,26 @@ let append Predicates = [IsRV32] in {
(NCLIPI (BuildGPRPair GPR:$lo, GPR:$hi), timm:$imm)>;
def : Pat<(XLenVT (riscv_nclipu XLenVT:$lo, XLenVT:$hi, timm:$imm)),
(NCLIPIU (BuildGPRPair GPR:$lo, GPR:$hi), timm:$imm)>;
+
+ // Packed "Q-format" multiply parts accumulate, scalar halfword (RV32).
+ def : Pat<(XLenVT (riscv_mqacc_h00 (XLenVT GPR:$rd),
+ (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
+ (MQACC_H00 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(XLenVT (riscv_mqacc_h01 (XLenVT GPR:$rd),
+ (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
+ (MQACC_H01 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(XLenVT (riscv_mqacc_h11 (XLenVT GPR:$rd),
+ (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
+ (MQACC_H11 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(XLenVT (riscv_mqracc_h00 (XLenVT GPR:$rd),
+ (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
+ (MQRACC_H00 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(XLenVT (riscv_mqracc_h01 (XLenVT GPR:$rd),
+ (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
+ (MQRACC_H01 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(XLenVT (riscv_mqracc_h11 (XLenVT GPR:$rd),
+ (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
+ (MQRACC_H11 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
} // append Predicates = [IsRV32]
let append Predicates = [IsRV64] in {
@@ -2751,6 +2815,64 @@ let append Predicates = [IsRV64] in {
def : Pat<(v2i32 (riscv_pnclipup_w GPR:$rs1, GPR:$rs2)),
(PNCLIPUP_W GPR:$rs1, GPR:$rs2)>;
+ // Packed "Q-format" multiply parts accumulate
+ def : Pat<(v2i32 (riscv_pmqacc_h00 (v2i32 GPR:$rd),
+ (v4i16 GPR:$rs1), (v4i16 GPR:$rs2))),
+ (PMQACC_W_H00 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(v2i32 (riscv_pmqacc_h01 (v2i32 GPR:$rd),
+ (v4i16 GPR:$rs1), (v4i16 GPR:$rs2))),
+ (PMQACC_W_H01 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(v2i32 (riscv_pmqacc_h11 (v2i32 GPR:$rd),
+ (v4i16 GPR:$rs1), (v4i16 GPR:$rs2))),
+ (PMQACC_W_H11 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(v2i32 (riscv_pmqracc_h00 (v2i32 GPR:$rd),
+ (v4i16 GPR:$rs1), (v4i16 GPR:$rs2))),
+ (PMQRACC_W_H00 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(v2i32 (riscv_pmqracc_h01 (v2i32 GPR:$rd),
+ (v4i16 GPR:$rs1), (v4i16 GPR:$rs2))),
+ (PMQRACC_W_H01 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(v2i32 (riscv_pmqracc_h11 (v2i32 GPR:$rd),
+ (v4i16 GPR:$rs1), (v4i16 GPR:$rs2))),
+ (PMQRACC_W_H11 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+
+ def : Pat<(XLenVT (riscv_mqacc_w00 (XLenVT GPR:$rd),
+ (v2i32 GPR:$rs1), (v2i32 GPR:$rs2))),
+ (MQACC_W00 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(XLenVT (riscv_mqacc_w01 (XLenVT GPR:$rd),
+ (v2i32 GPR:$rs1), (v2i32 GPR:$rs2))),
+ (MQACC_W01 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(XLenVT (riscv_mqacc_w11 (XLenVT GPR:$rd),
+ (v2i32 GPR:$rs1), (v2i32 GPR:$rs2))),
+ (MQACC_W11 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(XLenVT (riscv_mqracc_w00 (XLenVT GPR:$rd),
+ (v2i32 GPR:$rs1), (v2i32 GPR:$rs2))),
+ (MQRACC_W00 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(XLenVT (riscv_mqracc_w01 (XLenVT GPR:$rd),
+ (v2i32 GPR:$rs1), (v2i32 GPR:$rs2))),
+ (MQRACC_W01 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(XLenVT (riscv_mqracc_w11 (XLenVT GPR:$rd),
+ (v2i32 GPR:$rs1), (v2i32 GPR:$rs2))),
+ (MQRACC_W11 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+
+ def : Pat<(XLenVT (riscv_pmqacc_w_h00 (XLenVT GPR:$rd),
+ (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
+ (PMQACC_W_H00 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(XLenVT (riscv_pmqacc_w_h01 (XLenVT GPR:$rd),
+ (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
+ (PMQACC_W_H01 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(XLenVT (riscv_pmqacc_w_h11 (XLenVT GPR:$rd),
+ (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
+ (PMQACC_W_H11 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(XLenVT (riscv_pmqracc_w_h00 (XLenVT GPR:$rd),
+ (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
+ (PMQRACC_W_H00 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(XLenVT (riscv_pmqracc_w_h01 (XLenVT GPR:$rd),
+ (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
+ (PMQRACC_W_H01 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+ def : Pat<(XLenVT (riscv_pmqracc_w_h11 (XLenVT GPR:$rd),
+ (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
+ (PMQRACC_W_H11 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
+
def : Pat<(XLenVT (riscv_sati GPR:$rs1, timm:$imm)),
(SATI_RV64 GPR:$rs1, (IncImm timm:$imm))>;
def : Pat<(XLenVT (riscv_usati GPR:$rs1, timm:$imm)),
diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-32.ll b/llvm/test/CodeGen/RISCV/rvp-simd-32.ll
index d38012912872d..cebe60ef19249 100644
--- a/llvm/test/CodeGen/RISCV/rvp-simd-32.ll
+++ b/llvm/test/CodeGen/RISCV/rvp-simd-32.ll
@@ -3156,3 +3156,99 @@ define <2 x i16> @test_pnclipup_v2i16(i32 %a, i32 %b) {
%r = call <2 x i16> @llvm.riscv.pnclipup.v2i16.i32(i32 %a, i32 %b)
ret <2 x i16> %r
}
+
+define i32 @test_mqacc_h00_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) {
+; RV32-LABEL: test_mqacc_h00_i32:
+; RV32: # %bb.0:
+; RV32-NEXT: mqacc.h00 a0, a1, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_mqacc_h00_i32:
+; RV64: # %bb.0:
+; RV64-NEXT: zext.w a2, a2
+; RV64-NEXT: zext.w a1, a1
+; RV64-NEXT: pmqacc.w.h00 a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i32 @llvm.riscv.mqacc.h00.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_mqacc_h01_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) {
+; RV32-LABEL: test_mqacc_h01_i32:
+; RV32: # %bb.0:
+; RV32-NEXT: mqacc.h01 a0, a1, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_mqacc_h01_i32:
+; RV64: # %bb.0:
+; RV64-NEXT: zext.w a2, a2
+; RV64-NEXT: zext.w a1, a1
+; RV64-NEXT: pmqacc.w.h01 a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i32 @llvm.riscv.mqacc.h01.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_mqacc_h11_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) {
+; RV32-LABEL: test_mqacc_h11_i32:
+; RV32: # %bb.0:
+; RV32-NEXT: mqacc.h11 a0, a1, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_mqacc_h11_i32:
+; RV64: # %bb.0:
+; RV64-NEXT: zext.w a2, a2
+; RV64-NEXT: zext.w a1, a1
+; RV64-NEXT: pmqacc.w.h11 a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i32 @llvm.riscv.mqacc.h11.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_mqracc_h00_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) {
+; RV32-LABEL: test_mqracc_h00_i32:
+; RV32: # %bb.0:
+; RV32-NEXT: mqracc.h00 a0, a1, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_mqracc_h00_i32:
+; RV64: # %bb.0:
+; RV64-NEXT: zext.w a2, a2
+; RV64-NEXT: zext.w a1, a1
+; RV64-NEXT: pmqracc.w.h00 a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i32 @llvm.riscv.mqracc.h00.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_mqracc_h01_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) {
+; RV32-LABEL: test_mqracc_h01_i32:
+; RV32: # %bb.0:
+; RV32-NEXT: mqracc.h01 a0, a1, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_mqracc_h01_i32:
+; RV64: # %bb.0:
+; RV64-NEXT: zext.w a2, a2
+; RV64-NEXT: zext.w a1, a1
+; RV64-NEXT: pmqracc.w.h01 a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i32 @llvm.riscv.mqracc.h01.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
+
+define i32 @test_mqracc_h11_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) {
+; RV32-LABEL: test_mqracc_h11_i32:
+; RV32: # %bb.0:
+; RV32-NEXT: mqracc.h11 a0, a1, a2
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_mqracc_h11_i32:
+; RV64: # %bb.0:
+; RV64-NEXT: zext.w a2, a2
+; RV64-NEXT: zext.w a1, a1
+; RV64-NEXT: pmqracc.w.h11 a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i32 @llvm.riscv.mqracc.h11.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b)
+ ret i32 %r
+}
diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
index 96f16c3d52103..51b86057ccdbe 100644
--- a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
+++ b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll
@@ -6936,3 +6936,180 @@ define i64 @test_pnclipup_v2i32(i64 %a, i64 %b) {
%s = bitcast <2 x i32> %r to i64
ret i64 %s
}
+
+define <2 x i32> @test_pmqacc_h00_i32x2(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pmqacc_h00_i32x2:
+; RV32: # %bb.0:
+; RV32-NEXT: mqacc.h00 a1, a3, a5
+; RV32-NEXT: mqacc.h00 a0, a2, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pmqacc_h00_i32x2:
+; RV64: # %bb.0:
+; RV64-NEXT: pmqacc.w.h00 a0, a1, a2
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pmqacc.h00.v2i32.v4i16(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pmqacc_h01_i32x2(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pmqacc_h01_i32x2:
+; RV32: # %bb.0:
+; RV32-NEXT: mqacc.h01 a1, a3, a5
+; RV32-NEXT: mqacc.h01 a0, a2, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pmqacc_h01_i32x2:
+; RV64: # %bb.0:
+; RV64-NEXT: pmqacc.w.h01 a0, a1, a2
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pmqacc.h01.v2i32.v4i16(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pmqacc_h11_i32x2(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pmqacc_h11_i32x2:
+; RV32: # %bb.0:
+; RV32-NEXT: mqacc.h11 a1, a3, a5
+; RV32-NEXT: mqacc.h11 a0, a2, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pmqacc_h11_i32x2:
+; RV64: # %bb.0:
+; RV64-NEXT: pmqacc.w.h11 a0, a1, a2
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pmqacc.h11.v2i32.v4i16(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pmqracc_h00_i32x2(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pmqracc_h00_i32x2:
+; RV32: # %bb.0:
+; RV32-NEXT: mqracc.h00 a1, a3, a5
+; RV32-NEXT: mqracc.h00 a0, a2, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pmqracc_h00_i32x2:
+; RV64: # %bb.0:
+; RV64-NEXT: pmqracc.w.h00 a0, a1, a2
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pmqracc.h00.v2i32.v4i16(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pmqracc_h01_i32x2(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pmqracc_h01_i32x2:
+; RV32: # %bb.0:
+; RV32-NEXT: mqracc.h01 a1, a3, a5
+; RV32-NEXT: mqracc.h01 a0, a2, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pmqracc_h01_i32x2:
+; RV64: # %bb.0:
+; RV64-NEXT: pmqracc.w.h01 a0, a1, a2
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pmqracc.h01.v2i32.v4i16(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @test_pmqracc_h11_i32x2(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) {
+; RV32-LABEL: test_pmqracc_h11_i32x2:
+; RV32: # %bb.0:
+; RV32-NEXT: mqracc.h11 a1, a3, a5
+; RV32-NEXT: mqracc.h11 a0, a2, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_pmqracc_h11_i32x2:
+; RV64: # %bb.0:
+; RV64-NEXT: pmqracc.w.h11 a0, a1, a2
+; RV64-NEXT: ret
+ %r = call <2 x i32> @llvm.riscv.pmqracc.h11.v2i32.v4i16(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b)
+ ret <2 x i32> %r
+}
+
+; === Packed "Q-format" Multiply Parts Accumulate: word (i64 result) ===
+; RV64 uses mqacc.wXX; RV32 uses mqwacc/mqrwacc (register pair).
+
+define i64 @test_mqacc_w00_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_mqacc_w00_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mqwacc a0, a2, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_mqacc_w00_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: mqacc.w00 a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.mqacc.w00.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_mqacc_w01_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_mqacc_w01_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mqwacc a0, a2, a5
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_mqacc_w01_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: mqacc.w01 a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.mqacc.w01.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_mqacc_w11_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_mqacc_w11_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mqwacc a0, a3, a5
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_mqacc_w11_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: mqacc.w11 a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.mqacc.w11.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_mqracc_w00_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_mqracc_w00_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mqrwacc a0, a2, a4
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_mqracc_w00_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: mqracc.w00 a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.mqracc.w00.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_mqracc_w01_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_mqracc_w01_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mqrwacc a0, a2, a5
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_mqracc_w01_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: mqracc.w01 a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.mqracc.w01.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
+
+define i64 @test_mqracc_w11_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) {
+; RV32-LABEL: test_mqracc_w11_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: mqrwacc a0, a3, a5
+; RV32-NEXT: ret
+;
+; RV64-LABEL: test_mqracc_w11_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: mqracc.w11 a0, a1, a2
+; RV64-NEXT: ret
+ %r = call i64 @llvm.riscv.mqracc.w11.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b)
+ ret i64 %r
+}
>From 18ac2336d169764ad743bf24dde374ec5b9847ef Mon Sep 17 00:00:00 2001
From: Dongyan Chen <chendongyan at isrc.iscas.ac.cn>
Date: Sun, 23 Aug 2026 12:08:31 +0000
Subject: [PATCH 2/2] [RISCV][P-ext] resolve comment
---
clang/lib/Headers/riscv_packed_simd.h | 46 ++++++++---------
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 55 ++++++---------------
llvm/lib/Target/RISCV/RISCVInstrInfoP.td | 40 +++------------
llvm/test/CodeGen/RISCV/rvp-simd-32.ll | 20 ++------
4 files changed, 50 insertions(+), 111 deletions(-)
diff --git a/clang/lib/Headers/riscv_packed_simd.h b/clang/lib/Headers/riscv_packed_simd.h
index aaf7112d583f4..b5db8422a3381 100644
--- a/clang/lib/Headers/riscv_packed_simd.h
+++ b/clang/lib/Headers/riscv_packed_simd.h
@@ -285,7 +285,7 @@ typedef uint32_t uint32x2_t __attribute__((__vector_size__(8)));
return builtin(__rs1, __rs2); \
}
-#define __packed_ternary_builtin(name, rty, ty, builtin) \
+#define __packed_ternary_builtin_cast(name, rty, ty, builtin) \
static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(rty __rd, ty __rs1, \
ty __rs2) { \
return builtin(__rd, __rs1, __rs2); \
@@ -815,13 +815,13 @@ __packed_ternary_builtin_mixed(pmhraccsu_i32x2, int32x2_t, int32x2_t, uint32x2_t
/* Packed Absolute Difference Sum (32-bit) */
__packed_abdsum(pabdsumu_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumu_u8x4_u32)
-__packed_ternary_builtin(pabdsumau_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumau_u8x4_u32)
+__packed_ternary_builtin_cast(pabdsumau_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumau_u8x4_u32)
/* Packed Absolute Difference Sum (64-bit) */
__packed_abdsum(pabdsumu_u8x8_u32, uint32_t, uint8x8_t, __builtin_riscv_pabdsumu_u8x8_u32)
__packed_abdsum(pabdsumu_u8x8_u64, uint64_t, uint8x8_t, __builtin_riscv_pabdsumu_u8x8_u64)
-__packed_ternary_builtin(pabdsumau_u8x8_u32, uint32_t, uint8x8_t, __builtin_riscv_pabdsumau_u8x8_u32)
-__packed_ternary_builtin(pabdsumau_u8x8_u64, uint64_t, uint8x8_t, __builtin_riscv_pabdsumau_u8x8_u64)
+__packed_ternary_builtin_cast(pabdsumau_u8x8_u32, uint32_t, uint8x8_t, __builtin_riscv_pabdsumau_u8x8_u32)
+__packed_ternary_builtin_cast(pabdsumau_u8x8_u64, uint64_t, uint8x8_t, __builtin_riscv_pabdsumau_u8x8_u64)
/* Packed Saturating Absolute Value (32-bit) */
__packed_unary_builtin(psabs_i8x4, int8x4_t, __builtin_riscv_psabs_i8x4)
@@ -853,26 +853,26 @@ __packed_binary_builtin(pmulq_i32x2, int32x2_t, __builtin_riscv_pmulq_i32x2)
__packed_binary_builtin(pmulqr_i32x2, int32x2_t, __builtin_riscv_pmulqr_i32x2)
/* Packed "Q-format" Multiply Parts Accumulate (32-bit) */
-__packed_ternary_builtin(mqacc_h00_i32, int, int16x2_t, __builtin_riscv_mqacc_h00_i32)
-__packed_ternary_builtin(mqacc_h01_i32, int, int16x2_t, __builtin_riscv_mqacc_h01_i32)
-__packed_ternary_builtin(mqacc_h11_i32, int, int16x2_t, __builtin_riscv_mqacc_h11_i32)
-__packed_ternary_builtin(mqracc_h00_i32, int, int16x2_t, __builtin_riscv_mqracc_h00_i32)
-__packed_ternary_builtin(mqracc_h01_i32, int, int16x2_t, __builtin_riscv_mqracc_h01_i32)
-__packed_ternary_builtin(mqracc_h11_i32, int, int16x2_t, __builtin_riscv_mqracc_h11_i32)
+__packed_ternary_builtin_cast(mqacc_h00_i32, int, int16x2_t, __builtin_riscv_mqacc_h00_i32)
+__packed_ternary_builtin_cast(mqacc_h01_i32, int, int16x2_t, __builtin_riscv_mqacc_h01_i32)
+__packed_ternary_builtin_cast(mqacc_h11_i32, int, int16x2_t, __builtin_riscv_mqacc_h11_i32)
+__packed_ternary_builtin_cast(mqracc_h00_i32, int, int16x2_t, __builtin_riscv_mqracc_h00_i32)
+__packed_ternary_builtin_cast(mqracc_h01_i32, int, int16x2_t, __builtin_riscv_mqracc_h01_i32)
+__packed_ternary_builtin_cast(mqracc_h11_i32, int, int16x2_t, __builtin_riscv_mqracc_h11_i32)
/* Packed "Q-format" Multiply Parts Accumulate (64-bit) */
-__packed_ternary_builtin(pmqacc_h00_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqacc_h00_i32x2)
-__packed_ternary_builtin(pmqacc_h01_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqacc_h01_i32x2)
-__packed_ternary_builtin(pmqacc_h11_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqacc_h11_i32x2)
-__packed_ternary_builtin(pmqracc_h00_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqracc_h00_i32x2)
-__packed_ternary_builtin(pmqracc_h01_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqracc_h01_i32x2)
-__packed_ternary_builtin(pmqracc_h11_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqracc_h11_i32x2)
-__packed_ternary_builtin(mqacc_w00_i64, int64_t, int32x2_t, __builtin_riscv_mqacc_w00_i64)
-__packed_ternary_builtin(mqacc_w01_i64, int64_t, int32x2_t, __builtin_riscv_mqacc_w01_i64)
-__packed_ternary_builtin(mqacc_w11_i64, int64_t, int32x2_t, __builtin_riscv_mqacc_w11_i64)
-__packed_ternary_builtin(mqracc_w00_i64, int64_t, int32x2_t, __builtin_riscv_mqracc_w00_i64)
-__packed_ternary_builtin(mqracc_w01_i64, int64_t, int32x2_t, __builtin_riscv_mqracc_w01_i64)
-__packed_ternary_builtin(mqracc_w11_i64, int64_t, int32x2_t, __builtin_riscv_mqracc_w11_i64)
+__packed_ternary_builtin_cast(pmqacc_h00_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqacc_h00_i32x2)
+__packed_ternary_builtin_cast(pmqacc_h01_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqacc_h01_i32x2)
+__packed_ternary_builtin_cast(pmqacc_h11_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqacc_h11_i32x2)
+__packed_ternary_builtin_cast(pmqracc_h00_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqracc_h00_i32x2)
+__packed_ternary_builtin_cast(pmqracc_h01_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqracc_h01_i32x2)
+__packed_ternary_builtin_cast(pmqracc_h11_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqracc_h11_i32x2)
+__packed_ternary_builtin_cast(mqacc_w00_i64, int64_t, int32x2_t, __builtin_riscv_mqacc_w00_i64)
+__packed_ternary_builtin_cast(mqacc_w01_i64, int64_t, int32x2_t, __builtin_riscv_mqacc_w01_i64)
+__packed_ternary_builtin_cast(mqacc_w11_i64, int64_t, int32x2_t, __builtin_riscv_mqacc_w11_i64)
+__packed_ternary_builtin_cast(mqracc_w00_i64, int64_t, int32x2_t, __builtin_riscv_mqracc_w00_i64)
+__packed_ternary_builtin_cast(mqracc_w01_i64, int64_t, int32x2_t, __builtin_riscv_mqracc_w01_i64)
+__packed_ternary_builtin_cast(mqracc_w11_i64, int64_t, int32x2_t, __builtin_riscv_mqracc_w11_i64)
/* Packed Narrowing Clip Pair (32-bit) */
__packed_binary_builtin_cast(pnclipp_i8x4, int16x2_t, int8x4_t, __builtin_riscv_pnclipp_i8x4)
@@ -1034,7 +1034,7 @@ __packed_reinterpret(u32x2_i32x2, int32x2_t, uint32x2_t)
#undef __packed_nziph2
#undef __packed_nziph4
#undef __packed_abdsum
-#undef __packed_ternary_builtin
+#undef __packed_ternary_builtin_cast
#undef __packed_reinterpret
#undef __DEFAULT_FN_ATTRS
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 63d3522f88e47..c626487d73685 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -12289,16 +12289,22 @@ static unsigned getRVPQFormatAccOpcode(Intrinsic::ID IntNo) {
switch (IntNo) {
default:
llvm_unreachable("Unexpected RISC-V packed Q-format accumulate intrinsic");
+ case Intrinsic::riscv_mqacc_h00:
case Intrinsic::riscv_pmqacc_h00:
return RISCVISD::PMQACC_W_H00;
+ case Intrinsic::riscv_mqacc_h01:
case Intrinsic::riscv_pmqacc_h01:
return RISCVISD::PMQACC_W_H01;
+ case Intrinsic::riscv_mqacc_h11:
case Intrinsic::riscv_pmqacc_h11:
return RISCVISD::PMQACC_W_H11;
+ case Intrinsic::riscv_mqracc_h00:
case Intrinsic::riscv_pmqracc_h00:
return RISCVISD::PMQRACC_W_H00;
+ case Intrinsic::riscv_mqracc_h01:
case Intrinsic::riscv_pmqracc_h01:
return RISCVISD::PMQRACC_W_H01;
+ case Intrinsic::riscv_mqracc_h11:
case Intrinsic::riscv_pmqracc_h11:
return RISCVISD::PMQRACC_W_H11;
}
@@ -12323,25 +12329,6 @@ static unsigned getRVPQFormatAccWordOpcode(Intrinsic::ID IntNo) {
}
}
-static unsigned getRVPQFormatAccWideOpcode(Intrinsic::ID IntNo) {
- switch (IntNo) {
- default:
- llvm_unreachable("Unexpected RISC-V packed Q-format accumulate intrinsic");
- case Intrinsic::riscv_mqacc_h00:
- return RISCVISD::PMQACC_W_H00_W;
- case Intrinsic::riscv_mqacc_h01:
- return RISCVISD::PMQACC_W_H01_W;
- case Intrinsic::riscv_mqacc_h11:
- return RISCVISD::PMQACC_W_H11_W;
- case Intrinsic::riscv_mqracc_h00:
- return RISCVISD::PMQRACC_W_H00_W;
- case Intrinsic::riscv_mqracc_h01:
- return RISCVISD::PMQRACC_W_H01_W;
- case Intrinsic::riscv_mqracc_h11:
- return RISCVISD::PMQRACC_W_H11_W;
- }
-}
-
SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
SelectionDAG &DAG) const {
unsigned IntNo = Op.getConstantOperandVal(0);
@@ -12611,19 +12598,14 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
SDValue RdLo = DAG.getExtractVectorElt(DL, XLenVT, Rd, 0);
SDValue RdHi = DAG.getExtractVectorElt(DL, XLenVT, Rd, 1);
unsigned ScalarOpc = getRVPQFormatAccScalarOpcode(IntNo);
- SDValue Lo = DAG.getNode(ScalarOpc, DL, XLenVT, RdLo,
- DAG.getBitcast(XLenVT, Rs1Lo),
- DAG.getBitcast(XLenVT, Rs2Lo));
- SDValue Hi = DAG.getNode(ScalarOpc, DL, XLenVT, RdHi,
- DAG.getBitcast(XLenVT, Rs1Hi),
- DAG.getBitcast(XLenVT, Rs2Hi));
+ SDValue Lo = DAG.getNode(ScalarOpc, DL, XLenVT, RdLo, Rs1Lo, Rs2Lo);
+ SDValue Hi = DAG.getNode(ScalarOpc, DL, XLenVT, RdHi, Rs1Hi, Rs2Hi);
return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Lo, Hi);
}
if (VT == MVT::i32 && !Subtarget.is64Bit()) {
unsigned Opc = getRVPQFormatAccScalarOpcode(IntNo);
- return DAG.getNode(Opc, DL, XLenVT, Rd, DAG.getBitcast(XLenVT, Rs1),
- DAG.getBitcast(XLenVT, Rs2));
+ return DAG.getNode(Opc, DL, XLenVT, Rd, Rs1, Rs2);
}
if (VT == MVT::i64 && Subtarget.is64Bit()) {
@@ -17138,22 +17120,17 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
EVT VT = N->getValueType(0);
if (!Subtarget.is64Bit() || VT != MVT::i32)
return;
- MVT XLenVT = Subtarget.getXLenVT();
- // Widen a v2i16 source to XLenVT: bitcast to the 32-bit scalar it shares
- // bits with, then zero-extend into the upper half of the GPR.
+ SDValue Rd =
+ DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2i32, N->getOperand(1));
auto WidenSrc = [&](SDValue V) {
- if (V.getValueType() == XLenVT)
- return V;
- assert(V.getValueType() == MVT::v2i16);
- return DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT,
- DAG.getBitcast(MVT::i32, V));
+ return DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16,
+ {V, DAG.getUNDEF(MVT::v2i16)});
};
- SDValue Rd = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(1));
SDValue Rs1 = WidenSrc(N->getOperand(2));
SDValue Rs2 = WidenSrc(N->getOperand(3));
- unsigned Opc = getRVPQFormatAccWideOpcode(IntNo);
- SDValue Res = DAG.getNode(Opc, DL, XLenVT, Rd, Rs1, Rs2);
- Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
+ unsigned Opc = getRVPQFormatAccOpcode(IntNo);
+ SDValue Res = DAG.getNode(Opc, DL, MVT::v2i32, Rd, Rs1, Rs2);
+ Results.push_back(DAG.getExtractVectorElt(DL, MVT::i32, Res, 0));
return;
}
case Intrinsic::riscv_mqacc_w00:
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
index 799949da7ce2b..ef008ef0745da 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
@@ -1969,7 +1969,7 @@ def riscv_pnclipup_w : RVSDNode<"PNCLIPUP_W", SDT_RISCVPackedNarrowingClipW>;
// Packed "Q-format" multiply parts accumulate.
def SDT_RISCVQFormatAccS
: SDTypeProfile<1, 3, [SDTCisVT<0, XLenVT>, SDTCisVT<1, XLenVT>,
- SDTCisVT<2, XLenVT>,
+ SDTCisVT<2, v2i16>,
SDTCisSameAs<2, 3>]>;
def SDT_RISCVQFormatAccP
: SDTypeProfile<1, 3, [SDTCisVT<0, v2i32>, SDTCisVT<1, v2i32>,
@@ -1993,13 +1993,6 @@ def riscv_pmqracc_h00 : RVSDNode<"PMQRACC_W_H00", SDT_RISCVQFormatAccP>;
def riscv_pmqracc_h01 : RVSDNode<"PMQRACC_W_H01", SDT_RISCVQFormatAccP>;
def riscv_pmqracc_h11 : RVSDNode<"PMQRACC_W_H11", SDT_RISCVQFormatAccP>;
-def riscv_pmqacc_w_h00 : RVSDNode<"PMQACC_W_H00_W", SDT_RISCVQFormatAccS>;
-def riscv_pmqacc_w_h01 : RVSDNode<"PMQACC_W_H01_W", SDT_RISCVQFormatAccS>;
-def riscv_pmqacc_w_h11 : RVSDNode<"PMQACC_W_H11_W", SDT_RISCVQFormatAccS>;
-def riscv_pmqracc_w_h00 : RVSDNode<"PMQRACC_W_H00_W", SDT_RISCVQFormatAccS>;
-def riscv_pmqracc_w_h01 : RVSDNode<"PMQRACC_W_H01_W", SDT_RISCVQFormatAccS>;
-def riscv_pmqracc_w_h11 : RVSDNode<"PMQRACC_W_H11_W", SDT_RISCVQFormatAccS>;
-
def riscv_mqacc_w00 : RVSDNode<"MQACC_W00", SDT_RISCVQFormatAccW>;
def riscv_mqacc_w01 : RVSDNode<"MQACC_W01", SDT_RISCVQFormatAccW>;
def riscv_mqacc_w11 : RVSDNode<"MQACC_W11", SDT_RISCVQFormatAccW>;
@@ -2776,22 +2769,22 @@ let append Predicates = [IsRV32] in {
// Packed "Q-format" multiply parts accumulate, scalar halfword (RV32).
def : Pat<(XLenVT (riscv_mqacc_h00 (XLenVT GPR:$rd),
- (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
+ (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
(MQACC_H00 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
def : Pat<(XLenVT (riscv_mqacc_h01 (XLenVT GPR:$rd),
- (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
+ (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
(MQACC_H01 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
def : Pat<(XLenVT (riscv_mqacc_h11 (XLenVT GPR:$rd),
- (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
+ (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
(MQACC_H11 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
def : Pat<(XLenVT (riscv_mqracc_h00 (XLenVT GPR:$rd),
- (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
+ (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
(MQRACC_H00 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
def : Pat<(XLenVT (riscv_mqracc_h01 (XLenVT GPR:$rd),
- (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
+ (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
(MQRACC_H01 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
def : Pat<(XLenVT (riscv_mqracc_h11 (XLenVT GPR:$rd),
- (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
+ (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))),
(MQRACC_H11 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
} // append Predicates = [IsRV32]
@@ -2854,25 +2847,6 @@ let append Predicates = [IsRV64] in {
(v2i32 GPR:$rs1), (v2i32 GPR:$rs2))),
(MQRACC_W11 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
- def : Pat<(XLenVT (riscv_pmqacc_w_h00 (XLenVT GPR:$rd),
- (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
- (PMQACC_W_H00 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
- def : Pat<(XLenVT (riscv_pmqacc_w_h01 (XLenVT GPR:$rd),
- (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
- (PMQACC_W_H01 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
- def : Pat<(XLenVT (riscv_pmqacc_w_h11 (XLenVT GPR:$rd),
- (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
- (PMQACC_W_H11 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
- def : Pat<(XLenVT (riscv_pmqracc_w_h00 (XLenVT GPR:$rd),
- (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
- (PMQRACC_W_H00 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
- def : Pat<(XLenVT (riscv_pmqracc_w_h01 (XLenVT GPR:$rd),
- (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
- (PMQRACC_W_H01 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
- def : Pat<(XLenVT (riscv_pmqracc_w_h11 (XLenVT GPR:$rd),
- (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))),
- (PMQRACC_W_H11 GPR:$rd, GPR:$rs1, GPR:$rs2)>;
-
def : Pat<(XLenVT (riscv_sati GPR:$rs1, timm:$imm)),
(SATI_RV64 GPR:$rs1, (IncImm timm:$imm))>;
def : Pat<(XLenVT (riscv_usati GPR:$rs1, timm:$imm)),
diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-32.ll b/llvm/test/CodeGen/RISCV/rvp-simd-32.ll
index cebe60ef19249..f1209617b6a90 100644
--- a/llvm/test/CodeGen/RISCV/rvp-simd-32.ll
+++ b/llvm/test/CodeGen/RISCV/rvp-simd-32.ll
@@ -2408,10 +2408,10 @@ define <2 x i16> @test_select_v2i16(i1 %cond, <2 x i16> %a, <2 x i16> %b) {
; CHECK: # %bb.0:
; CHECK-NEXT: andi a3, a0, 1
; CHECK-NEXT: mv a0, a1
-; CHECK-NEXT: bnez a3, [[SELECT_BB:.LBB[0-9]+_2]]
+; CHECK-NEXT: bnez a3, .LBB170_2
; CHECK-NEXT: # %bb.1:
; CHECK-NEXT: mv a0, a2
-; CHECK-NEXT: [[SELECT_BB]]:
+; CHECK-NEXT: .LBB170_2:
; CHECK-NEXT: ret
%res = select i1 %cond, <2 x i16> %a, <2 x i16> %b
ret <2 x i16> %res
@@ -2422,10 +2422,10 @@ define <4 x i8> @test_select_v4i8(i1 %cond, <4 x i8> %a, <4 x i8> %b) {
; CHECK: # %bb.0:
; CHECK-NEXT: andi a3, a0, 1
; CHECK-NEXT: mv a0, a1
-; CHECK-NEXT: bnez a3, [[SELECT_BB:.LBB[0-9]+_2]]
+; CHECK-NEXT: bnez a3, .LBB171_2
; CHECK-NEXT: # %bb.1:
; CHECK-NEXT: mv a0, a2
-; CHECK-NEXT: [[SELECT_BB]]:
+; CHECK-NEXT: .LBB171_2:
; CHECK-NEXT: ret
%res = select i1 %cond, <4 x i8> %a, <4 x i8> %b
ret <4 x i8> %res
@@ -3165,8 +3165,6 @@ define i32 @test_mqacc_h00_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) {
;
; RV64-LABEL: test_mqacc_h00_i32:
; RV64: # %bb.0:
-; RV64-NEXT: zext.w a2, a2
-; RV64-NEXT: zext.w a1, a1
; RV64-NEXT: pmqacc.w.h00 a0, a1, a2
; RV64-NEXT: ret
%r = call i32 @llvm.riscv.mqacc.h00.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b)
@@ -3181,8 +3179,6 @@ define i32 @test_mqacc_h01_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) {
;
; RV64-LABEL: test_mqacc_h01_i32:
; RV64: # %bb.0:
-; RV64-NEXT: zext.w a2, a2
-; RV64-NEXT: zext.w a1, a1
; RV64-NEXT: pmqacc.w.h01 a0, a1, a2
; RV64-NEXT: ret
%r = call i32 @llvm.riscv.mqacc.h01.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b)
@@ -3197,8 +3193,6 @@ define i32 @test_mqacc_h11_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) {
;
; RV64-LABEL: test_mqacc_h11_i32:
; RV64: # %bb.0:
-; RV64-NEXT: zext.w a2, a2
-; RV64-NEXT: zext.w a1, a1
; RV64-NEXT: pmqacc.w.h11 a0, a1, a2
; RV64-NEXT: ret
%r = call i32 @llvm.riscv.mqacc.h11.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b)
@@ -3213,8 +3207,6 @@ define i32 @test_mqracc_h00_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) {
;
; RV64-LABEL: test_mqracc_h00_i32:
; RV64: # %bb.0:
-; RV64-NEXT: zext.w a2, a2
-; RV64-NEXT: zext.w a1, a1
; RV64-NEXT: pmqracc.w.h00 a0, a1, a2
; RV64-NEXT: ret
%r = call i32 @llvm.riscv.mqracc.h00.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b)
@@ -3229,8 +3221,6 @@ define i32 @test_mqracc_h01_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) {
;
; RV64-LABEL: test_mqracc_h01_i32:
; RV64: # %bb.0:
-; RV64-NEXT: zext.w a2, a2
-; RV64-NEXT: zext.w a1, a1
; RV64-NEXT: pmqracc.w.h01 a0, a1, a2
; RV64-NEXT: ret
%r = call i32 @llvm.riscv.mqracc.h01.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b)
@@ -3245,8 +3235,6 @@ define i32 @test_mqracc_h11_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) {
;
; RV64-LABEL: test_mqracc_h11_i32:
; RV64: # %bb.0:
-; RV64-NEXT: zext.w a2, a2
-; RV64-NEXT: zext.w a1, a1
; RV64-NEXT: pmqracc.w.h11 a0, a1, a2
; RV64-NEXT: ret
%r = call i32 @llvm.riscv.mqracc.h11.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b)
More information about the cfe-commits
mailing list