[clang] [llvm] [RISCV] Support Packed Multiply High Accumulate (PR #217591)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 20 04:02:55 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-clang-codegen
Author: Hongyu Chen (XChy)
<details>
<summary>Changes</summary>
This patch adds the packed multiply high accumulate intrinsics and corresponding codegen. See also https://github.com/riscv/riscv-p-spec/blob/master/P-ext-intrinsics.adoc#packed-multiply-high-accumulate.
---
Patch is 33.30 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/217591.diff
9 Files Affected:
- (modified) clang/include/clang/Basic/BuiltinsRISCV.td (+22)
- (modified) clang/lib/CodeGen/TargetBuiltins/RISCV.cpp (+49)
- (modified) clang/lib/Headers/riscv_packed_simd.h (+36)
- (modified) cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c (+130)
- (modified) llvm/include/llvm/IR/IntrinsicsRISCV.td (+8)
- (modified) llvm/lib/Target/RISCV/RISCVISelLowering.cpp (+83)
- (modified) llvm/lib/Target/RISCV/RISCVInstrInfoP.td (+40)
- (modified) llvm/test/CodeGen/RISCV/rvp-simd-32.ll (+55)
- (modified) llvm/test/CodeGen/RISCV/rvp-simd-64.ll (+181)
``````````diff
diff --git a/clang/include/clang/Basic/BuiltinsRISCV.td b/clang/include/clang/Basic/BuiltinsRISCV.td
index b91b356d7a25d..09b43292d42b6 100644
--- a/clang/include/clang/Basic/BuiltinsRISCV.td
+++ b/clang/include/clang/Basic/BuiltinsRISCV.td
@@ -255,6 +255,14 @@ def pmulhru_u16x2 : RISCVBuiltin<"_Vector<2, unsigned short>(_Vector<2, unsigned
def pmulhsu_i16x2 : RISCVBuiltin<"_Vector<2, short>(_Vector<2, short>, _Vector<2, unsigned short>)">;
def pmulhrsu_i16x2 : RISCVBuiltin<"_Vector<2, short>(_Vector<2, short>, _Vector<2, unsigned short>)">;
+// Packed Multiply High Accumulate (32-bit)
+def pmhacc_i16x2 : RISCVBuiltin<"_Vector<2, short>(_Vector<2, short>, _Vector<2, short>, _Vector<2, short>)">;
+def pmhracc_i16x2 : RISCVBuiltin<"_Vector<2, short>(_Vector<2, short>, _Vector<2, short>, _Vector<2, short>)">;
+def pmhaccu_u16x2 : RISCVBuiltin<"_Vector<2, unsigned short>(_Vector<2, unsigned short>, _Vector<2, unsigned short>, _Vector<2, unsigned short>)">;
+def pmhraccu_u16x2 : RISCVBuiltin<"_Vector<2, unsigned short>(_Vector<2, unsigned short>, _Vector<2, unsigned short>, _Vector<2, unsigned short>)">;
+def pmhaccsu_i16x2 : RISCVBuiltin<"_Vector<2, short>(_Vector<2, short>, _Vector<2, short>, _Vector<2, unsigned short>)">;
+def pmhraccsu_i16x2 : RISCVBuiltin<"_Vector<2, short>(_Vector<2, short>, _Vector<2, short>, _Vector<2, unsigned short>)">;
+
// Packed Multiply High (64-bit)
def pmulh_i16x4 : RISCVBuiltin<"_Vector<4, short>(_Vector<4, short>, _Vector<4, short>)">;
def pmulh_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<2, int>)">;
@@ -269,6 +277,20 @@ def pmulhsu_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<2, un
def pmulhrsu_i16x4 : RISCVBuiltin<"_Vector<4, short>(_Vector<4, short>, _Vector<4, unsigned short>)">;
def pmulhrsu_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<2, unsigned int>)">;
+// Packed Multiply High Accumulate (64-bit)
+def pmhacc_i16x4 : RISCVBuiltin<"_Vector<4, short>(_Vector<4, short>, _Vector<4, short>, _Vector<4, short>)">;
+def pmhacc_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<2, int>, _Vector<2, int>)">;
+def pmhracc_i16x4 : RISCVBuiltin<"_Vector<4, short>(_Vector<4, short>, _Vector<4, short>, _Vector<4, short>)">;
+def pmhracc_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<2, int>, _Vector<2, int>)">;
+def pmhaccu_u16x4 : RISCVBuiltin<"_Vector<4, unsigned short>(_Vector<4, unsigned short>, _Vector<4, unsigned short>, _Vector<4, unsigned short>)">;
+def pmhaccu_u32x2 : RISCVBuiltin<"_Vector<2, unsigned int>(_Vector<2, unsigned int>, _Vector<2, unsigned int>, _Vector<2, unsigned int>)">;
+def pmhraccu_u16x4 : RISCVBuiltin<"_Vector<4, unsigned short>(_Vector<4, unsigned short>, _Vector<4, unsigned short>, _Vector<4, unsigned short>)">;
+def pmhraccu_u32x2 : RISCVBuiltin<"_Vector<2, unsigned int>(_Vector<2, unsigned int>, _Vector<2, unsigned int>, _Vector<2, unsigned int>)">;
+def pmhaccsu_i16x4 : RISCVBuiltin<"_Vector<4, short>(_Vector<4, short>, _Vector<4, short>, _Vector<4, unsigned short>)">;
+def pmhaccsu_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<2, int>, _Vector<2, unsigned int>)">;
+def pmhraccsu_i16x4 : RISCVBuiltin<"_Vector<4, short>(_Vector<4, short>, _Vector<4, short>, _Vector<4, unsigned short>)">;
+def pmhraccsu_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<2, int>, _Vector<2, unsigned int>)">;
+
// Packed Absolute Difference Sum (32-bit)
def pabdsumu_u8x4_u32 : RISCVBuiltin<"unsigned int(_Vector<4, unsigned char>, _Vector<4, unsigned char>)">;
def pabdsumau_u8x4_u32 : RISCVBuiltin<"unsigned int(unsigned int, _Vector<4, unsigned char>, _Vector<4, unsigned char>)">;
diff --git a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
index eb98b38974b21..1429510519ac0 100644
--- a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp
@@ -1278,6 +1278,25 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID,
case RISCV::BI__builtin_riscv_pmulhrsu_i16x2:
case RISCV::BI__builtin_riscv_pmulhrsu_i16x4:
case RISCV::BI__builtin_riscv_pmulhrsu_i32x2:
+ // Packed Multiply High Accumulate
+ case RISCV::BI__builtin_riscv_pmhacc_i16x2:
+ case RISCV::BI__builtin_riscv_pmhacc_i16x4:
+ case RISCV::BI__builtin_riscv_pmhacc_i32x2:
+ case RISCV::BI__builtin_riscv_pmhracc_i16x2:
+ case RISCV::BI__builtin_riscv_pmhracc_i16x4:
+ case RISCV::BI__builtin_riscv_pmhracc_i32x2:
+ case RISCV::BI__builtin_riscv_pmhaccu_u16x2:
+ case RISCV::BI__builtin_riscv_pmhaccu_u16x4:
+ case RISCV::BI__builtin_riscv_pmhaccu_u32x2:
+ case RISCV::BI__builtin_riscv_pmhraccu_u16x2:
+ case RISCV::BI__builtin_riscv_pmhraccu_u16x4:
+ case RISCV::BI__builtin_riscv_pmhraccu_u32x2:
+ case RISCV::BI__builtin_riscv_pmhaccsu_i16x2:
+ case RISCV::BI__builtin_riscv_pmhaccsu_i16x4:
+ case RISCV::BI__builtin_riscv_pmhaccsu_i32x2:
+ case RISCV::BI__builtin_riscv_pmhraccsu_i16x2:
+ case RISCV::BI__builtin_riscv_pmhraccsu_i16x4:
+ case RISCV::BI__builtin_riscv_pmhraccsu_i32x2:
// Packed Saturating Absolute Value
case RISCV::BI__builtin_riscv_psabs_i8x4:
case RISCV::BI__builtin_riscv_psabs_i16x2:
@@ -1412,6 +1431,36 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID,
case RISCV::BI__builtin_riscv_pmulhrsu_i32x2:
ID = Intrinsic::riscv_pmulhrsu;
break;
+ case RISCV::BI__builtin_riscv_pmhacc_i16x2:
+ case RISCV::BI__builtin_riscv_pmhacc_i16x4:
+ case RISCV::BI__builtin_riscv_pmhacc_i32x2:
+ ID = Intrinsic::riscv_pmhacc;
+ break;
+ case RISCV::BI__builtin_riscv_pmhracc_i16x2:
+ case RISCV::BI__builtin_riscv_pmhracc_i16x4:
+ case RISCV::BI__builtin_riscv_pmhracc_i32x2:
+ ID = Intrinsic::riscv_pmhracc;
+ break;
+ case RISCV::BI__builtin_riscv_pmhaccu_u16x2:
+ case RISCV::BI__builtin_riscv_pmhaccu_u16x4:
+ case RISCV::BI__builtin_riscv_pmhaccu_u32x2:
+ ID = Intrinsic::riscv_pmhaccu;
+ break;
+ case RISCV::BI__builtin_riscv_pmhraccu_u16x2:
+ case RISCV::BI__builtin_riscv_pmhraccu_u16x4:
+ case RISCV::BI__builtin_riscv_pmhraccu_u32x2:
+ ID = Intrinsic::riscv_pmhraccu;
+ break;
+ case RISCV::BI__builtin_riscv_pmhaccsu_i16x2:
+ case RISCV::BI__builtin_riscv_pmhaccsu_i16x4:
+ case RISCV::BI__builtin_riscv_pmhaccsu_i32x2:
+ ID = Intrinsic::riscv_pmhaccsu;
+ break;
+ case RISCV::BI__builtin_riscv_pmhraccsu_i16x2:
+ case RISCV::BI__builtin_riscv_pmhraccsu_i16x4:
+ case RISCV::BI__builtin_riscv_pmhraccsu_i32x2:
+ ID = Intrinsic::riscv_pmhraccsu;
+ break;
case RISCV::BI__builtin_riscv_psabs_i8x4:
case RISCV::BI__builtin_riscv_psabs_i16x2:
case RISCV::BI__builtin_riscv_psabs_i8x8:
diff --git a/clang/lib/Headers/riscv_packed_simd.h b/clang/lib/Headers/riscv_packed_simd.h
index 491e219bec297..0c0bcb2b88270 100644
--- a/clang/lib/Headers/riscv_packed_simd.h
+++ b/clang/lib/Headers/riscv_packed_simd.h
@@ -77,6 +77,18 @@ typedef uint32_t uint32x2_t __attribute__((__vector_size__(8)));
return builtin(__rs1, __rs2); \
}
+#define __packed_ternary_builtin(name, ty, builtin) \
+ static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name( \
+ ty __rd, ty __rs1, ty __rs2) { \
+ return builtin(__rd, __rs1, __rs2); \
+ }
+
+#define __packed_ternary_builtin_mixed(name, rty, ty1, ty2, builtin) \
+ static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name( \
+ rty __rd, ty1 __rs1, ty2 __rs2) { \
+ return builtin(__rd, __rs1, __rs2); \
+ }
+
#define __packed_sh1add(name, ty) \
static __inline__ ty __DEFAULT_FN_ATTRS __riscv_##name(ty __rs1, ty __rs2) { \
return (__rs1 << 1) + __rs2; \
@@ -728,6 +740,14 @@ __packed_binary_builtin(pmulhru_u16x2, uint16x2_t, __builtin_riscv_pmulhru_u16x2
__packed_binary_builtin_mixed(pmulhsu_i16x2, int16x2_t, int16x2_t, uint16x2_t, __builtin_riscv_pmulhsu_i16x2)
__packed_binary_builtin_mixed(pmulhrsu_i16x2, int16x2_t, int16x2_t, uint16x2_t, __builtin_riscv_pmulhrsu_i16x2)
+/* Packed Multiply High Accumulate (32-bit) */
+__packed_ternary_builtin(pmhacc_i16x2, int16x2_t, __builtin_riscv_pmhacc_i16x2)
+__packed_ternary_builtin(pmhracc_i16x2, int16x2_t, __builtin_riscv_pmhracc_i16x2)
+__packed_ternary_builtin(pmhaccu_u16x2, uint16x2_t, __builtin_riscv_pmhaccu_u16x2)
+__packed_ternary_builtin(pmhraccu_u16x2, uint16x2_t, __builtin_riscv_pmhraccu_u16x2)
+__packed_ternary_builtin_mixed(pmhaccsu_i16x2, int16x2_t, int16x2_t, uint16x2_t, __builtin_riscv_pmhaccsu_i16x2)
+__packed_ternary_builtin_mixed(pmhraccsu_i16x2, int16x2_t, int16x2_t, uint16x2_t, __builtin_riscv_pmhraccsu_i16x2)
+
/* Packed Multiply High (64-bit) */
__packed_binary_builtin(pmulh_i16x4, int16x4_t, __builtin_riscv_pmulh_i16x4)
__packed_binary_builtin(pmulh_i32x2, int32x2_t, __builtin_riscv_pmulh_i32x2)
@@ -742,6 +762,20 @@ __packed_binary_builtin_mixed(pmulhsu_i32x2, int32x2_t, int32x2_t, uint32x2_t, _
__packed_binary_builtin_mixed(pmulhrsu_i16x4, int16x4_t, int16x4_t, uint16x4_t, __builtin_riscv_pmulhrsu_i16x4)
__packed_binary_builtin_mixed(pmulhrsu_i32x2, int32x2_t, int32x2_t, uint32x2_t, __builtin_riscv_pmulhrsu_i32x2)
+/* Packed Multiply High Accumulate (64-bit) */
+__packed_ternary_builtin(pmhacc_i16x4, int16x4_t, __builtin_riscv_pmhacc_i16x4)
+__packed_ternary_builtin(pmhacc_i32x2, int32x2_t, __builtin_riscv_pmhacc_i32x2)
+__packed_ternary_builtin(pmhracc_i16x4, int16x4_t, __builtin_riscv_pmhracc_i16x4)
+__packed_ternary_builtin(pmhracc_i32x2, int32x2_t, __builtin_riscv_pmhracc_i32x2)
+__packed_ternary_builtin(pmhaccu_u16x4, uint16x4_t, __builtin_riscv_pmhaccu_u16x4)
+__packed_ternary_builtin(pmhaccu_u32x2, uint32x2_t, __builtin_riscv_pmhaccu_u32x2)
+__packed_ternary_builtin(pmhraccu_u16x4, uint16x4_t, __builtin_riscv_pmhraccu_u16x4)
+__packed_ternary_builtin(pmhraccu_u32x2, uint32x2_t, __builtin_riscv_pmhraccu_u32x2)
+__packed_ternary_builtin_mixed(pmhaccsu_i16x4, int16x4_t, int16x4_t, uint16x4_t, __builtin_riscv_pmhaccsu_i16x4)
+__packed_ternary_builtin_mixed(pmhaccsu_i32x2, int32x2_t, int32x2_t, uint32x2_t, __builtin_riscv_pmhaccsu_i32x2)
+__packed_ternary_builtin_mixed(pmhraccsu_i16x4, int16x4_t, int16x4_t, uint16x4_t, __builtin_riscv_pmhraccsu_i16x4)
+__packed_ternary_builtin_mixed(pmhraccsu_i32x2, int32x2_t, int32x2_t, uint32x2_t, __builtin_riscv_pmhraccsu_i32x2)
+
/* Packed Absolute Difference Sum (32-bit) */
__packed_abdsum(pabdsumu_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumu_u8x4_u32)
__packed_abdsum_acc(pabdsumau_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumau_u8x4_u32)
@@ -900,6 +934,8 @@ __packed_reinterpret(u32x2_i32x2, int32x2_t, uint32x2_t)
#undef __packed_unary_op
#undef __packed_binary_builtin
#undef __packed_binary_builtin_mixed
+#undef __packed_ternary_builtin
+#undef __packed_ternary_builtin_mixed
#undef __packed_sh1add
#undef __packed_sh1sadd
#undef __packed_cmp
diff --git a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
index eac091c66bba0..9c76e228e92bf 100644
--- a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
+++ b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c
@@ -2502,6 +2502,136 @@ int32x2_t test_pmulhrsu_i32x2(int32x2_t rs1, uint32x2_t rs2) {
return __riscv_pmulhrsu_i32x2(rs1, rs2);
}
+// CHECK-LABEL: test_pmhacc_i16x2:
+// CHECK: pmhacc.h
+int16x2_t test_pmhacc_i16x2(int16x2_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pmhacc_i16x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmhracc_i16x2:
+// CHECK: pmhracc.h
+int16x2_t test_pmhracc_i16x2(int16x2_t rd, int16x2_t rs1, int16x2_t rs2) {
+ return __riscv_pmhracc_i16x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmhaccu_u16x2:
+// CHECK: pmhaccu.h
+uint16x2_t test_pmhaccu_u16x2(uint16x2_t rd, uint16x2_t rs1,
+ uint16x2_t rs2) {
+ return __riscv_pmhaccu_u16x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmhraccu_u16x2:
+// CHECK: pmhraccu.h
+uint16x2_t test_pmhraccu_u16x2(uint16x2_t rd, uint16x2_t rs1,
+ uint16x2_t rs2) {
+ return __riscv_pmhraccu_u16x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmhaccsu_i16x2:
+// CHECK: pmhaccsu.h
+int16x2_t test_pmhaccsu_i16x2(int16x2_t rd, int16x2_t rs1,
+ uint16x2_t rs2) {
+ return __riscv_pmhaccsu_i16x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmhraccsu_i16x2:
+// CHECK: pmhraccsu.h
+int16x2_t test_pmhraccsu_i16x2(int16x2_t rd, int16x2_t rs1,
+ uint16x2_t rs2) {
+ return __riscv_pmhraccsu_i16x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmhacc_i16x4:
+// RV32-COUNT-2: pmhacc.h
+// RV64: pmhacc.h
+int16x4_t test_pmhacc_i16x4(int16x4_t rd, int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pmhacc_i16x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmhracc_i16x4:
+// RV32-COUNT-2: pmhracc.h
+// RV64: pmhracc.h
+int16x4_t test_pmhracc_i16x4(int16x4_t rd, int16x4_t rs1, int16x4_t rs2) {
+ return __riscv_pmhracc_i16x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmhaccu_u16x4:
+// RV32-COUNT-2: pmhaccu.h
+// RV64: pmhaccu.h
+uint16x4_t test_pmhaccu_u16x4(uint16x4_t rd, uint16x4_t rs1,
+ uint16x4_t rs2) {
+ return __riscv_pmhaccu_u16x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmhraccu_u16x4:
+// RV32-COUNT-2: pmhraccu.h
+// RV64: pmhraccu.h
+uint16x4_t test_pmhraccu_u16x4(uint16x4_t rd, uint16x4_t rs1,
+ uint16x4_t rs2) {
+ return __riscv_pmhraccu_u16x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmhaccsu_i16x4:
+// RV32-COUNT-2: pmhaccsu.h
+// RV64: pmhaccsu.h
+int16x4_t test_pmhaccsu_i16x4(int16x4_t rd, int16x4_t rs1,
+ uint16x4_t rs2) {
+ return __riscv_pmhaccsu_i16x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmhraccsu_i16x4:
+// RV32-COUNT-2: pmhraccsu.h
+// RV64: pmhraccsu.h
+int16x4_t test_pmhraccsu_i16x4(int16x4_t rd, int16x4_t rs1,
+ uint16x4_t rs2) {
+ return __riscv_pmhraccsu_i16x4(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmhacc_i32x2:
+// RV32-COUNT-2: mhacc
+// RV64: pmhacc.w
+int32x2_t test_pmhacc_i32x2(int32x2_t rd, int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_pmhacc_i32x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmhracc_i32x2:
+// RV32-COUNT-2: mhracc
+// RV64: pmhracc.w
+int32x2_t test_pmhracc_i32x2(int32x2_t rd, int32x2_t rs1, int32x2_t rs2) {
+ return __riscv_pmhracc_i32x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmhaccu_u32x2:
+// RV32-COUNT-2: mhaccu
+// RV64: pmhaccu.w
+uint32x2_t test_pmhaccu_u32x2(uint32x2_t rd, uint32x2_t rs1,
+ uint32x2_t rs2) {
+ return __riscv_pmhaccu_u32x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmhraccu_u32x2:
+// RV32-COUNT-2: mhraccu
+// RV64: pmhraccu.w
+uint32x2_t test_pmhraccu_u32x2(uint32x2_t rd, uint32x2_t rs1,
+ uint32x2_t rs2) {
+ return __riscv_pmhraccu_u32x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmhaccsu_i32x2:
+// RV32-COUNT-2: mhaccsu
+// RV64: pmhaccsu.w
+int32x2_t test_pmhaccsu_i32x2(int32x2_t rd, int32x2_t rs1, uint32x2_t rs2) {
+ return __riscv_pmhaccsu_i32x2(rd, rs1, rs2);
+}
+
+// CHECK-LABEL: test_pmhraccsu_i32x2:
+// RV32-COUNT-2: mhraccsu
+// RV64: pmhraccsu.w
+int32x2_t test_pmhraccsu_i32x2(int32x2_t rd, int32x2_t rs1, uint32x2_t rs2) {
+ return __riscv_pmhraccsu_i32x2(rd, rs1, rs2);
+}
+
// CHECK-LABEL: test_prev_i8x4:
// CHECK: rev8
int8x4_t test_prev_i8x4(int8x4_t a) { return __riscv_prev_i8x4(a); }
diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td
index d5bd6243bd1a6..8fa0ee78ca2a5 100644
--- a/llvm/include/llvm/IR/IntrinsicsRISCV.td
+++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td
@@ -2117,6 +2117,14 @@ class RVPBinaryIntrinsic
def int_riscv_pmulhsu : RVPBinaryIntrinsic;
def int_riscv_pmulhrsu : RVPBinaryIntrinsic;
+ // Packed Multiply High Accumulate.
+ def int_riscv_pmhacc : RVPTernaryIntrinsic;
+ def int_riscv_pmhracc : RVPTernaryIntrinsic;
+ def int_riscv_pmhaccu : RVPTernaryIntrinsic;
+ def int_riscv_pmhraccu : RVPTernaryIntrinsic;
+ def int_riscv_pmhaccsu : RVPTernaryIntrinsic;
+ def int_riscv_pmhraccsu : RVPTernaryIntrinsic;
+
// Packed Absolute Difference Sum.
def int_riscv_pabdsumu
: DefaultAttrsIntrinsic<[llvm_anyint_ty],
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 6bd0a66767e6b..baaf7e40199e8 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -12134,6 +12134,26 @@ static unsigned getRVPMulHighOpcode(unsigned IntNo) {
}
}
+static unsigned getRVPMulHighAccumulateOpcode(unsigned IntNo) {
+ switch (IntNo) {
+ default:
+ llvm_unreachable(
+ "Unexpected RISC-V packed multiply high accumulate intrinsic");
+ case Intrinsic::riscv_pmhacc:
+ return RISCVISD::PMHACC;
+ case Intrinsic::riscv_pmhracc:
+ return RISCVISD::PMHRACC;
+ case Intrinsic::riscv_pmhaccu:
+ return RISCVISD::PMHACCU;
+ case Intrinsic::riscv_pmhraccu:
+ return RISCVISD::PMHRACCU;
+ case Intrinsic::riscv_pmhaccsu:
+ return RISCVISD::PMHACCSU;
+ case Intrinsic::riscv_pmhraccsu:
+ return RISCVISD::PMHRACCSU;
+ }
+}
+
SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
SelectionDAG &DAG) const {
unsigned IntNo = Op.getConstantOperandVal(0);
@@ -12486,6 +12506,45 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
return DAG.getNode(Opc, DL, VT, Op.getOperand(1), Op.getOperand(2));
}
+ case Intrinsic::riscv_pmhacc:
+ case Intrinsic::riscv_pmhracc:
+ case Intrinsic::riscv_pmhaccu:
+ case Intrinsic::riscv_pmhraccu:
+ case Intrinsic::riscv_pmhaccsu:
+ case Intrinsic::riscv_pmhraccsu: {
+ EVT VT = Op.getValueType();
+ unsigned MulOpc = getRVPMulHighAccumulateOpcode(IntNo);
+ SDValue Rd = Op.getOperand(1);
+ SDValue Rs1 = Op.getOperand(2);
+ SDValue Rs2 = Op.getOperand(3);
+
+ // RV32 has no single instruction for 64-bit packed multiply high
+ // accumulate. Split v4i16 into two v2i16 packed operations, and split
+ // v2i32 into scalar i32 operations.
+ if (!Subtarget.is64Bit() && VT == MVT::v4i16) {
+ auto [RdLo, RdHi] = DAG.SplitVector(Rd, DL);
+ auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
+ auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
+ SDValue Lo =
+ DAG.getNode(MulOpc, DL, MVT::v2i16, RdLo, Rs1Lo, Rs2Lo);
+ SDValue Hi =
+ DAG.getNode(MulOpc, DL, MVT::v2i16, RdHi, Rs1Hi, Rs2Hi);
+ return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
+ }
+
+ if (!Subtarget.is64Bit() && VT == MVT::v2i32) {
+ auto Extract = [&](SDValue V, unsigned Idx) {
+ return DAG.getExtractVectorElt(DL, MVT::i32, V, Idx);
+ };
+ SDValue Lo = DAG.getNode(MulOpc, DL, MVT::i32, Extract(Rd, 0),
+ Extract(Rs1, 0), Extract(Rs2, 0));
+ SDValue Hi = DAG.getNode(MulOpc, DL, MVT::i32, Extract(Rd, 1),
+ Extract(Rs1, 1), Extract(Rs2, 1));
+ return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Lo, Hi);
+ }
+
+ return DAG.getNode(MulOpc, DL, VT, Rd, Rs1, Rs2);
+ }
case Intrinsic::riscv_pmerge: {
EVT VT = Op.getValueType();
auto buildMerge = [&](SDValue Rs1, SDValue Rs2, SDValue Mask,
@@ -16554,6 +16613,30 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
Results.push_back(DAG.getExtractSubvector(DL, VT, Res, 0));
return;
}
+ case Intrinsic::riscv_pmhacc:
+ case Intrinsic::riscv_pmhracc:
+ case Intrinsic::riscv_pmhaccu:
+ case Intrinsic::riscv_pmhraccu:
+ case Intrinsic::riscv_pmhaccsu:
+ case Intrinsic::riscv_pmhraccsu: {
+ EVT VT = N->getValueType(0);
+ if (!Subtarget.is64Bit() || VT != MVT::v2i16)
+ return;
+
+ EVT WideVT = MV...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/217591
More information about the llvm-commits
mailing list