[llvm] [LLVM][CodeGen][SVE] Use pairwise instruction for partial_reduce_[s/u]mla. (PR #212772)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 29 06:42:16 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Paul Walker (paulwalker-arm)
<details>
<summary>Changes</summary>
[S/U]ADALP can process all elements, which saves an instruction over the existing pair of top/bottom wide adds, assuming the predicate can be ignored. The accumulating pairwise instructions also have a fast path for the accumulation which might reduce end-to-end latency as well.
---
Full diff: https://github.com/llvm/llvm-project/pull/212772.diff
4 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td (+14-13)
- (modified) llvm/test/CodeGen/AArch64/sve-partial-reduce-dot-product.ll (+5-8)
- (modified) llvm/test/CodeGen/AArch64/sve-partial-reduce-wide-add.ll (+22-24)
- (modified) llvm/test/CodeGen/AArch64/vector-absolute-difference.ll (+4-4)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index d926c4023acb3..78e02063d4458 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4048,6 +4048,20 @@ let Predicates = [HasSVE2_or_SME] in {
defm SADALP_ZPmZ : sve2_int_sadd_long_accum_pairwise<0, "sadalp", int_aarch64_sve_sadalp>;
defm UADALP_ZPmZ : sve2_int_sadd_long_accum_pairwise<1, "uadalp", int_aarch64_sve_uadalp>;
+ def : Pat<(nxv8i16 (partial_reduce_umla nxv8i16:$Acc, nxv16i8:$Input, (nxv16i8 (splat_vector (i32 1))))),
+ (UADALP_ZPmZ_H (PTRUE_H 31), $Acc, $Input)>;
+ def : Pat<(nxv4i32 (partial_reduce_umla nxv4i32:$Acc, nxv8i16:$Input, (nxv8i16 (splat_vector (i32 1))))),
+ (UADALP_ZPmZ_S (PTRUE_S 31), $Acc, $Input)>;
+ def : Pat<(nxv2i64 (partial_reduce_umla nxv2i64:$Acc, nxv4i32:$Input, (nxv4i32 (splat_vector (i32 1))))),
+ (UADALP_ZPmZ_D (PTRUE_D 31), $Acc, $Input)>;
+
+ def : Pat<(nxv8i16 (partial_reduce_smla nxv8i16:$Acc, nxv16i8:$Input, (nxv16i8 (splat_vector (i32 1))))),
+ (SADALP_ZPmZ_H (PTRUE_H 31), $Acc, $Input)>;
+ def : Pat<(nxv4i32 (partial_reduce_smla nxv4i32:$Acc, nxv8i16:$Input, (nxv8i16 (splat_vector (i32 1))))),
+ (SADALP_ZPmZ_S (PTRUE_S 31), $Acc, $Input)>;
+ def : Pat<(nxv2i64 (partial_reduce_smla nxv2i64:$Acc, nxv4i32:$Input, (nxv4i32 (splat_vector (i32 1))))),
+ (SADALP_ZPmZ_D (PTRUE_D 31), $Acc, $Input)>;
+
// SVE2 integer pairwise arithmetic
defm ADDP_ZPmZ : sve2_int_arith_pred<0b100011, "addp", int_aarch64_sve_addp>;
defm SMAXP_ZPmZ : sve2_int_arith_pred<0b101001, "smaxp", int_aarch64_sve_smaxp>;
@@ -4136,19 +4150,6 @@ let Predicates = [HasSVE2_or_SME] in {
defm USUBWB_ZZZ : sve2_wide_int_arith_wide<0b110, "usubwb", int_aarch64_sve_usubwb>;
defm USUBWT_ZZZ : sve2_wide_int_arith_wide<0b111, "usubwt", int_aarch64_sve_usubwt>;
- def : Pat<(nxv2i64 (partial_reduce_umla nxv2i64:$Acc, nxv4i32:$Input, (nxv4i32 (splat_vector (i32 1))))),
- (UADDWT_ZZZ_D (UADDWB_ZZZ_D $Acc, $Input), $Input)>;
- def : Pat<(nxv2i64 (partial_reduce_smla nxv2i64:$Acc, nxv4i32:$Input, (nxv4i32 (splat_vector (i32 1))))),
- (SADDWT_ZZZ_D (SADDWB_ZZZ_D $Acc, $Input), $Input)>;
- def : Pat<(nxv4i32 (partial_reduce_umla nxv4i32:$Acc, nxv8i16:$Input, (nxv8i16 (splat_vector (i32 1))))),
- (UADDWT_ZZZ_S (UADDWB_ZZZ_S $Acc, $Input), $Input)>;
- def : Pat<(nxv4i32 (partial_reduce_smla nxv4i32:$Acc, nxv8i16:$Input, (nxv8i16 (splat_vector (i32 1))))),
- (SADDWT_ZZZ_S (SADDWB_ZZZ_S $Acc, $Input), $Input)>;
- def : Pat<(nxv8i16 (partial_reduce_umla nxv8i16:$Acc, nxv16i8:$Input, (nxv16i8 (splat_vector (i32 1))))),
- (UADDWT_ZZZ_H (UADDWB_ZZZ_H $Acc, $Input), $Input)>;
- def : Pat<(nxv8i16 (partial_reduce_smla nxv8i16:$Acc, nxv16i8:$Input, (nxv16i8 (splat_vector (i32 1))))),
- (SADDWT_ZZZ_H (SADDWB_ZZZ_H $Acc, $Input), $Input)>;
-
// SVE2 integer multiply long
defm SQDMULLB_ZZZ : sve2_wide_int_arith_long<0b11000, "sqdmullb", int_aarch64_sve_sqdmullb>;
defm SQDMULLT_ZZZ : sve2_wide_int_arith_long<0b11001, "sqdmullt", int_aarch64_sve_sqdmullt>;
diff --git a/llvm/test/CodeGen/AArch64/sve-partial-reduce-dot-product.ll b/llvm/test/CodeGen/AArch64/sve-partial-reduce-dot-product.ll
index 3f97b74db5afd..836a22b3d69f6 100644
--- a/llvm/test/CodeGen/AArch64/sve-partial-reduce-dot-product.ll
+++ b/llvm/test/CodeGen/AArch64/sve-partial-reduce-dot-product.ll
@@ -1324,18 +1324,15 @@ define <2 x i32> @udot_v16i8tov2i32(<2 x i32> %acc, <16 x i8> %input) "target-fe
; CHECK-SME-LABEL: udot_v16i8tov2i32:
; CHECK-SME: // %bb.0: // %entry
; CHECK-SME-NEXT: uunpklo z2.h, z1.b
+; CHECK-SME-NEXT: ptrue p0.s
; CHECK-SME-NEXT: ext z1.b, z1.b, z1.b, #8
; CHECK-SME-NEXT: uunpklo z1.h, z1.b
-; CHECK-SME-NEXT: uaddwb z0.s, z0.s, z2.h
-; CHECK-SME-NEXT: uaddwt z0.s, z0.s, z2.h
+; CHECK-SME-NEXT: uadalp z0.s, p0/m, z2.h
; CHECK-SME-NEXT: ext z2.b, z2.b, z2.b, #8
-; CHECK-SME-NEXT: uaddwb z0.s, z0.s, z2.h
-; CHECK-SME-NEXT: uaddwt z0.s, z0.s, z2.h
-; CHECK-SME-NEXT: uaddwb z0.s, z0.s, z1.h
-; CHECK-SME-NEXT: uaddwt z0.s, z0.s, z1.h
+; CHECK-SME-NEXT: uadalp z0.s, p0/m, z2.h
+; CHECK-SME-NEXT: uadalp z0.s, p0/m, z1.h
; CHECK-SME-NEXT: ext z1.b, z1.b, z1.b, #8
-; CHECK-SME-NEXT: uaddwb z0.s, z0.s, z1.h
-; CHECK-SME-NEXT: uaddwt z0.s, z0.s, z1.h
+; CHECK-SME-NEXT: uadalp z0.s, p0/m, z1.h
; CHECK-SME-NEXT: ret
entry:
%input.wide = zext <16 x i8> %input to <16 x i32>
diff --git a/llvm/test/CodeGen/AArch64/sve-partial-reduce-wide-add.ll b/llvm/test/CodeGen/AArch64/sve-partial-reduce-wide-add.ll
index cf738b61a01ee..c7cb0bb43aea3 100644
--- a/llvm/test/CodeGen/AArch64/sve-partial-reduce-wide-add.ll
+++ b/llvm/test/CodeGen/AArch64/sve-partial-reduce-wide-add.ll
@@ -13,8 +13,8 @@ define <vscale x 2 x i64> @signed_wide_add_nxv4i32(<vscale x 2 x i64> %acc, <vsc
;
; CHECK-SVE2-LABEL: signed_wide_add_nxv4i32:
; CHECK-SVE2: // %bb.0: // %entry
-; CHECK-SVE2-NEXT: saddwb z0.d, z0.d, z1.s
-; CHECK-SVE2-NEXT: saddwt z0.d, z0.d, z1.s
+; CHECK-SVE2-NEXT: ptrue p0.d
+; CHECK-SVE2-NEXT: sadalp z0.d, p0/m, z1.s
; CHECK-SVE2-NEXT: ret
entry:
%input.wide = sext <vscale x 4 x i32> %input to <vscale x 4 x i64>
@@ -33,8 +33,8 @@ define <vscale x 2 x i64> @unsigned_wide_add_nxv4i32(<vscale x 2 x i64> %acc, <v
;
; CHECK-SVE2-LABEL: unsigned_wide_add_nxv4i32:
; CHECK-SVE2: // %bb.0: // %entry
-; CHECK-SVE2-NEXT: uaddwb z0.d, z0.d, z1.s
-; CHECK-SVE2-NEXT: uaddwt z0.d, z0.d, z1.s
+; CHECK-SVE2-NEXT: ptrue p0.d
+; CHECK-SVE2-NEXT: uadalp z0.d, p0/m, z1.s
; CHECK-SVE2-NEXT: ret
entry:
%input.wide = zext <vscale x 4 x i32> %input to <vscale x 4 x i64>
@@ -53,8 +53,8 @@ define <vscale x 4 x i32> @signed_wide_add_nxv8i16(<vscale x 4 x i32> %acc, <vsc
;
; CHECK-SVE2-LABEL: signed_wide_add_nxv8i16:
; CHECK-SVE2: // %bb.0: // %entry
-; CHECK-SVE2-NEXT: saddwb z0.s, z0.s, z1.h
-; CHECK-SVE2-NEXT: saddwt z0.s, z0.s, z1.h
+; CHECK-SVE2-NEXT: ptrue p0.s
+; CHECK-SVE2-NEXT: sadalp z0.s, p0/m, z1.h
; CHECK-SVE2-NEXT: ret
entry:
%input.wide = sext <vscale x 8 x i16> %input to <vscale x 8 x i32>
@@ -73,8 +73,8 @@ define <vscale x 4 x i32> @unsigned_wide_add_nxv8i16(<vscale x 4 x i32> %acc, <v
;
; CHECK-SVE2-LABEL: unsigned_wide_add_nxv8i16:
; CHECK-SVE2: // %bb.0: // %entry
-; CHECK-SVE2-NEXT: uaddwb z0.s, z0.s, z1.h
-; CHECK-SVE2-NEXT: uaddwt z0.s, z0.s, z1.h
+; CHECK-SVE2-NEXT: ptrue p0.s
+; CHECK-SVE2-NEXT: uadalp z0.s, p0/m, z1.h
; CHECK-SVE2-NEXT: ret
entry:
%input.wide = zext <vscale x 8 x i16> %input to <vscale x 8 x i32>
@@ -93,8 +93,8 @@ define <vscale x 8 x i16> @signed_wide_add_nxv16i8(<vscale x 8 x i16> %acc, <vsc
;
; CHECK-SVE2-LABEL: signed_wide_add_nxv16i8:
; CHECK-SVE2: // %bb.0: // %entry
-; CHECK-SVE2-NEXT: saddwb z0.h, z0.h, z1.b
-; CHECK-SVE2-NEXT: saddwt z0.h, z0.h, z1.b
+; CHECK-SVE2-NEXT: ptrue p0.h
+; CHECK-SVE2-NEXT: sadalp z0.h, p0/m, z1.b
; CHECK-SVE2-NEXT: ret
entry:
%input.wide = sext <vscale x 16 x i8> %input to <vscale x 16 x i16>
@@ -113,8 +113,8 @@ define <vscale x 8 x i16> @unsigned_wide_add_nxv16i8(<vscale x 8 x i16> %acc, <v
;
; CHECK-SVE2-LABEL: unsigned_wide_add_nxv16i8:
; CHECK-SVE2: // %bb.0: // %entry
-; CHECK-SVE2-NEXT: uaddwb z0.h, z0.h, z1.b
-; CHECK-SVE2-NEXT: uaddwt z0.h, z0.h, z1.b
+; CHECK-SVE2-NEXT: ptrue p0.h
+; CHECK-SVE2-NEXT: uadalp z0.h, p0/m, z1.b
; CHECK-SVE2-NEXT: ret
entry:
%input.wide = zext <vscale x 16 x i8> %input to <vscale x 16 x i16>
@@ -137,8 +137,8 @@ define <vscale x 2 x i32> @signed_wide_add_nxv4i16(<vscale x 2 x i32> %acc, <vsc
; CHECK-SVE2: // %bb.0: // %entry
; CHECK-SVE2-NEXT: ptrue p0.s
; CHECK-SVE2-NEXT: sxth z1.s, p0/m, z1.s
-; CHECK-SVE2-NEXT: saddwb z0.d, z0.d, z1.s
-; CHECK-SVE2-NEXT: saddwt z0.d, z0.d, z1.s
+; CHECK-SVE2-NEXT: ptrue p0.d
+; CHECK-SVE2-NEXT: sadalp z0.d, p0/m, z1.s
; CHECK-SVE2-NEXT: ret
entry:
%input.wide = sext <vscale x 4 x i16> %input to <vscale x 4 x i32>
@@ -159,8 +159,8 @@ define <vscale x 2 x i32> @unsigned_wide_add_nxv4i16(<vscale x 2 x i32> %acc, <v
; CHECK-SVE2-LABEL: unsigned_wide_add_nxv4i16:
; CHECK-SVE2: // %bb.0: // %entry
; CHECK-SVE2-NEXT: and z1.s, z1.s, #0xffff
-; CHECK-SVE2-NEXT: uaddwb z0.d, z0.d, z1.s
-; CHECK-SVE2-NEXT: uaddwt z0.d, z0.d, z1.s
+; CHECK-SVE2-NEXT: ptrue p0.d
+; CHECK-SVE2-NEXT: uadalp z0.d, p0/m, z1.s
; CHECK-SVE2-NEXT: ret
entry:
%input.wide = zext <vscale x 4 x i16> %input to <vscale x 4 x i32>
@@ -183,10 +183,9 @@ define <vscale x 4 x i64> @signed_wide_add_nxv8i32(<vscale x 4 x i64> %acc, <vsc
;
; CHECK-SVE2-LABEL: signed_wide_add_nxv8i32:
; CHECK-SVE2: // %bb.0: // %entry
-; CHECK-SVE2-NEXT: saddwb z1.d, z1.d, z3.s
-; CHECK-SVE2-NEXT: saddwb z0.d, z0.d, z2.s
-; CHECK-SVE2-NEXT: saddwt z1.d, z1.d, z3.s
-; CHECK-SVE2-NEXT: saddwt z0.d, z0.d, z2.s
+; CHECK-SVE2-NEXT: ptrue p0.d
+; CHECK-SVE2-NEXT: sadalp z0.d, p0/m, z2.s
+; CHECK-SVE2-NEXT: sadalp z1.d, p0/m, z3.s
; CHECK-SVE2-NEXT: ret
entry:
%input.wide = sext <vscale x 8 x i32> %input to <vscale x 8 x i64>
@@ -209,10 +208,9 @@ define <vscale x 4 x i64> @unsigned_wide_add_nxv8i32(<vscale x 4 x i64> %acc, <v
;
; CHECK-SVE2-LABEL: unsigned_wide_add_nxv8i32:
; CHECK-SVE2: // %bb.0: // %entry
-; CHECK-SVE2-NEXT: uaddwb z1.d, z1.d, z3.s
-; CHECK-SVE2-NEXT: uaddwb z0.d, z0.d, z2.s
-; CHECK-SVE2-NEXT: uaddwt z1.d, z1.d, z3.s
-; CHECK-SVE2-NEXT: uaddwt z0.d, z0.d, z2.s
+; CHECK-SVE2-NEXT: ptrue p0.d
+; CHECK-SVE2-NEXT: uadalp z0.d, p0/m, z2.s
+; CHECK-SVE2-NEXT: uadalp z1.d, p0/m, z3.s
; CHECK-SVE2-NEXT: ret
entry:
%input.wide = zext <vscale x 8 x i32> %input to <vscale x 8 x i64>
diff --git a/llvm/test/CodeGen/AArch64/vector-absolute-difference.ll b/llvm/test/CodeGen/AArch64/vector-absolute-difference.ll
index bebf1cb00e6a9..45a4b403fe391 100644
--- a/llvm/test/CodeGen/AArch64/vector-absolute-difference.ll
+++ b/llvm/test/CodeGen/AArch64/vector-absolute-difference.ll
@@ -33,8 +33,8 @@ define <vscale x 8 x i16> @sabs_nxv16i8_wide_add(<vscale x 8 x i16> %acc, <vscal
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b
; CHECK-NEXT: sabd z1.b, p0/m, z1.b, z2.b
-; CHECK-NEXT: uaddwb z0.h, z0.h, z1.b
-; CHECK-NEXT: uaddwt z0.h, z0.h, z1.b
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: uadalp z0.h, p0/m, z1.b
; CHECK-NEXT: ret
%smax = tail call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
%smin = tail call <vscale x 16 x i8> @llvm.smin.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
@@ -88,8 +88,8 @@ define <vscale x 4 x i32> @uabs_nxv16i8_wide_add(<vscale x 4 x i32> %acc, <vscal
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.h
; CHECK-NEXT: uabd z1.h, p0/m, z1.h, z2.h
-; CHECK-NEXT: uaddwb z0.s, z0.s, z1.h
-; CHECK-NEXT: uaddwt z0.s, z0.s, z1.h
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: uadalp z0.s, p0/m, z1.h
; CHECK-NEXT: ret
%umax = tail call <vscale x 8 x i16> @llvm.umax.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
%umin = tail call <vscale x 8 x i16> @llvm.umin.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
``````````
</details>
https://github.com/llvm/llvm-project/pull/212772
More information about the llvm-commits
mailing list