[llvm] 6c18428 - [AArch64][SVE2p1] Add 2-way SVE2p1 dot product intrinsics
David Sherwood via llvm-commits
llvm-commits at lists.llvm.org
Fri Feb 3 04:08:07 PST 2023
Author: David Sherwood
Date: 2023-02-03T12:07:53Z
New Revision: 6c1842845fdc4bf50f15dd774085ca4781fe1fbd
URL: https://github.com/llvm/llvm-project/commit/6c1842845fdc4bf50f15dd774085ca4781fe1fbd
DIFF: https://github.com/llvm/llvm-project/commit/6c1842845fdc4bf50f15dd774085ca4781fe1fbd.diff
LOG: [AArch64][SVE2p1] Add 2-way SVE2p1 dot product intrinsics
This patch adds the LLVM IR intrinsics for the following:
* sdot (2-way, vectors + indexed)
* udot (2-way, vectors + indexed)
* fdot (vectors + indexed)
Differential Revision: https://reviews.llvm.org/D143066
Added:
llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dots.ll
Modified:
llvm/include/llvm/IR/IntrinsicsAArch64.td
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
llvm/lib/Target/AArch64/SVEInstrFormats.td
Removed:
################################################################################
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 0d5917e1ad65..18604eff71bf 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -3219,4 +3219,12 @@ let TargetPrefix = "aarch64" in {
def int_aarch64_sve_uzpq_x2 : SVE2_VG2_ZipUzp_Intrinsic;
def int_aarch64_sve_uzp_x4 : SVE2_VG4_ZipUzp_Intrinsic;
def int_aarch64_sve_uzpq_x4 : SVE2_VG4_ZipUzp_Intrinsic;
+
+ // Vector dot-products (2-way)
+ def int_aarch64_sve_sdot_x2 : SVE2_3VectorArg_Long_Intrinsic;
+ def int_aarch64_sve_udot_x2 : SVE2_3VectorArg_Long_Intrinsic;
+ def int_aarch64_sve_fdot_x2 : SVE2_3VectorArg_Long_Intrinsic;
+ def int_aarch64_sve_sdot_lane_x2 : SVE2_3VectorArgIndexed_Long_Intrinsic;
+ def int_aarch64_sve_udot_lane_x2 : SVE2_3VectorArgIndexed_Long_Intrinsic;
+ def int_aarch64_sve_fdot_lane_x2 : SVE2_3VectorArgIndexed_Long_Intrinsic;
}
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 4b55d6f196f0..275fdeea4128 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -3690,17 +3690,17 @@ defm PSEL_PPPRI : sve2_int_perm_sel_p<"psel", int_aarch64_sve_psel>;
let Predicates = [HasSVE2p1_or_HasSME2] in {
defm FCLAMP_ZZZ : sve2p1_fclamp<"fclamp", int_aarch64_sve_fclamp>;
-def FDOT_ZZZ_S : sve_float_dot<0b0, "fdot">;
-def FDOT_ZZZI_S : sve_float_dot_indexed<0b0, "fdot">;
+defm FDOT_ZZZ_S : sve_float_dot<0b0, "fdot", nxv8f16, int_aarch64_sve_fdot_x2>;
+defm FDOT_ZZZI_S : sve_float_dot_indexed<0b0, "fdot", nxv8f16, int_aarch64_sve_fdot_lane_x2>;
def BFMLSLB_ZZZ_S : sve2_fp_mla_long<0b110, "bfmlslb">;
def BFMLSLT_ZZZ_S : sve2_fp_mla_long<0b111, "bfmlslt">;
def BFMLSLB_ZZZI_S : sve2_fp_mla_long_by_indexed_elem<0b110, "bfmlslb">;
def BFMLSLT_ZZZI_S : sve2_fp_mla_long_by_indexed_elem<0b111, "bfmlslt">;
-def SDOT_ZZZ_HtoS : sve2p1_two_way_dot_vv<"sdot", 0b0>;
-def UDOT_ZZZ_HtoS : sve2p1_two_way_dot_vv<"udot", 0b1>;
-def SDOT_ZZZI_HtoS : sve2p1_two_way_dot_vvi<"sdot", 0b0>;
-def UDOT_ZZZI_HtoS : sve2p1_two_way_dot_vvi<"udot", 0b1>;
+defm SDOT_ZZZ_HtoS : sve2p1_two_way_dot_vv<"sdot", 0b0, int_aarch64_sve_sdot_x2>;
+defm UDOT_ZZZ_HtoS : sve2p1_two_way_dot_vv<"udot", 0b1, int_aarch64_sve_udot_x2>;
+defm SDOT_ZZZI_HtoS : sve2p1_two_way_dot_vvi<"sdot", 0b0, int_aarch64_sve_sdot_lane_x2>;
+defm UDOT_ZZZI_HtoS : sve2p1_two_way_dot_vvi<"udot", 0b1, int_aarch64_sve_udot_lane_x2>;
defm CNTP_XCI : sve2p1_pcount_pn<"cntp", 0b000>;
defm PEXT_PCI : sve2p1_pred_as_ctr_to_mask<"pext">;
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 17816f425c72..cb2a52a045fa 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -9036,6 +9036,11 @@ class sve2p1_two_way_dot_vv<string mnemonic, bit u>
let hasSideEffects = 0;
}
+multiclass sve2p1_two_way_dot_vv<string mnemonic, bit u, SDPatternOperator intrinsic> {
+ def NAME : sve2p1_two_way_dot_vv<mnemonic, u>;
+
+ def : SVE_3_Op_Pat<nxv4i32, intrinsic, nxv4i32, nxv8i16, nxv8i16, !cast<Instruction>(NAME)>;
+}
// SVE two-way dot product (indexed)
class sve2p1_two_way_dot_vvi<string mnemonic, bit u>
@@ -9059,6 +9064,11 @@ class sve2p1_two_way_dot_vvi<string mnemonic, bit u>
let hasSideEffects = 0;
}
+multiclass sve2p1_two_way_dot_vvi<string mnemonic, bit u, SDPatternOperator intrinsic> {
+ def NAME : sve2p1_two_way_dot_vvi<mnemonic, u>;
+
+ def : SVE_4_Op_Imm_Pat<nxv4i32, intrinsic, nxv4i32, nxv8i16, nxv8i16, i32, VectorIndexS32b_timm, !cast<Instruction>(NAME)>;
+}
class sve2p1_ptrue_pn<string mnemonic, bits<2> sz, PNRP8to15RegOp pnrty>
: I<(outs pnrty:$PNd), (ins ), mnemonic, "\t$PNd",
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dots.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dots.ll
new file mode 100644
index 000000000000..d8c8043df59c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-dots.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+
+define <vscale x 4 x i32> @sdot_x2(<vscale x 4 x i32> %zda, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm) {
+; CHECK-LABEL: sdot_x2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sdot z0.s, z1.h, z2.h
+; CHECK-NEXT: ret
+ %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.x2.nxv4i32(<vscale x 4 x i32> %zda, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm)
+ ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 4 x i32> @udot_x2(<vscale x 4 x i32> %zda, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm) {
+; CHECK-LABEL: udot_x2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: udot z0.s, z1.h, z2.h
+; CHECK-NEXT: ret
+ %out = call <vscale x 4 x i32> @llvm.aarch64.sve.udot.x2.nxv4i32(<vscale x 4 x i32> %zda, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm)
+ ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 4 x float> @fdot_x2(<vscale x 4 x float> %zda, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm) {
+; CHECK-LABEL: fdot_x2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fdot z0.s, z1.h, z2.h
+; CHECK-NEXT: ret
+ %out = call <vscale x 4 x float> @llvm.aarch64.sve.fdot.x2.nxv4f32(<vscale x 4 x float> %zda, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm)
+ ret <vscale x 4 x float> %out
+}
+
+define <vscale x 4 x i32> @sdot_lane_x2(<vscale x 4 x i32> %zda, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm) {
+; CHECK-LABEL: sdot_lane_x2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sdot z0.s, z1.h, z2.h[3]
+; CHECK-NEXT: ret
+ %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.lane.x2.nxv4i32(<vscale x 4 x i32> %zda, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm, i32 3)
+ ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 4 x i32> @udot_lane_x2(<vscale x 4 x i32> %zda, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm) {
+; CHECK-LABEL: udot_lane_x2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: udot z0.s, z1.h, z2.h[3]
+; CHECK-NEXT: ret
+ %out = call <vscale x 4 x i32> @llvm.aarch64.sve.udot.lane.x2.nxv4i32(<vscale x 4 x i32> %zda, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm, i32 3)
+ ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 4 x float> @fdot_lane_x2(<vscale x 4 x float> %zda, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm) {
+; CHECK-LABEL: fdot_lane_x2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fdot z0.s, z1.h, z2.h[3]
+; CHECK-NEXT: ret
+ %out = call <vscale x 4 x float> @llvm.aarch64.sve.fdot.lane.x2.nxv4f32(<vscale x 4 x float> %zda, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm, i32 3)
+ ret <vscale x 4 x float> %out
+}
+
+
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sdot.x2.nxv4i32(<vscale x 4 x i32> %zda, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.udot.x2.nxv4i32(<vscale x 4 x i32> %zda, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fdot.x2.nxv4f32(<vscale x 4 x float> %zda, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.sdot.lane.x2.nxv4i32(<vscale x 4 x i32> %zda, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm, i32)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.udot.lane.x2.nxv4i32(<vscale x 4 x i32> %zda, <vscale x 8 x i16> %zn, <vscale x 8 x i16> %zm, i32)
+declare <vscale x 4 x float> @llvm.aarch64.sve.fdot.lane.x2.nxv4f32(<vscale x 4 x float> %zda, <vscale x 8 x half> %zn, <vscale x 8 x half> %zm, i32)
More information about the llvm-commits
mailing list