[llvm] [LLVM][CodeGen][SVE] Add isel patterns for abs-diff partial reductions. (PR #212800)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 29 08:43:51 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Paul Walker (paulwalker-arm)
<details>
<summary>Changes</summary>
For SVE2 we can use top/bottom [S/U]ABAL[T/B] instruction. SVE2p3 can use just [S/U]ABAL.
When updating the tests for https://github.com/llvm/llvm-project/pull/212772 I noticed the TODO, which I figured I may was well fix given I'm in the area.
---
Full diff: https://github.com/llvm/llvm-project/pull/212800.diff
2 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td (+28)
- (modified) llvm/test/CodeGen/AArch64/vector-absolute-difference.ll (+106-25)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index d926c4023acb3..6a464616fdf0e 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4186,6 +4186,20 @@ let Predicates = [HasSVE2_or_SME] in {
defm UABALB_ZZZ : sve2_int_absdiff_accum_long<0b10, "uabalb", int_aarch64_sve_uabalb>;
defm UABALT_ZZZ : sve2_int_absdiff_accum_long<0b11, "uabalt", int_aarch64_sve_uabalt>;
+ def : Pat<(nxv8i16 (partial_reduce_umla nxv8i16:$Acc, (AArch64sabd_p (SVEAnyPredicate), nxv16i8:$Op1, nxv16i8:$Op2), (nxv16i8 (splat_vector (i32 1))))),
+ (SABALT_ZZZ_H (SABALB_ZZZ_H $Acc, $Op1, $Op2), $Op1, $Op2)>;
+ def : Pat<(nxv4i32 (partial_reduce_umla nxv4i32:$Acc, (AArch64sabd_p (SVEAnyPredicate), nxv8i16:$Op1, nxv8i16:$Op2), (nxv8i16 (splat_vector (i32 1))))),
+ (SABALT_ZZZ_S (SABALB_ZZZ_S $Acc, $Op1, $Op2), $Op1, $Op2)>;
+ def : Pat<(nxv2i64 (partial_reduce_umla nxv2i64:$Acc, (AArch64sabd_p (SVEAnyPredicate), nxv4i32:$Op1, nxv4i32:$Op2), (nxv4i32 (splat_vector (i32 1))))),
+ (SABALT_ZZZ_D (SABALB_ZZZ_D $Acc, $Op1, $Op2), $Op1, $Op2)>;
+
+ def : Pat<(nxv8i16 (partial_reduce_umla nxv8i16:$Acc, (AArch64uabd_p (SVEAnyPredicate), nxv16i8:$Op1, nxv16i8:$Op2), (nxv16i8 (splat_vector (i32 1))))),
+ (UABALT_ZZZ_H (UABALB_ZZZ_H $Acc, $Op1, $Op2), $Op1, $Op2)>;
+ def : Pat<(nxv4i32 (partial_reduce_umla nxv4i32:$Acc, (AArch64uabd_p (SVEAnyPredicate), nxv8i16:$Op1, nxv8i16:$Op2), (nxv8i16 (splat_vector (i32 1))))),
+ (UABALT_ZZZ_S (UABALB_ZZZ_S $Acc, $Op1, $Op2), $Op1, $Op2)>;
+ def : Pat<(nxv2i64 (partial_reduce_umla nxv2i64:$Acc, (AArch64uabd_p (SVEAnyPredicate), nxv4i32:$Op1, nxv4i32:$Op2), (nxv4i32 (splat_vector (i32 1))))),
+ (UABALT_ZZZ_D (UABALB_ZZZ_D $Acc, $Op1, $Op2), $Op1, $Op2)>;
+
// SVE2 integer add/subtract long with carry
defm ADCLB_ZZZ : sve2_int_addsub_long_carry<0b00, "adclb", int_aarch64_sve_adclb>;
defm ADCLT_ZZZ : sve2_int_addsub_long_carry<0b01, "adclt", int_aarch64_sve_adclt>;
@@ -4965,6 +4979,20 @@ let Predicates = [HasSVE2p3_or_SME2p3] in {
defm SABAL_ZZZ : sve2_int_two_way_absdiff_accum_long<0b0, "sabal", int_aarch64_sve_sabal>;
defm UABAL_ZZZ : sve2_int_two_way_absdiff_accum_long<0b1, "uabal", int_aarch64_sve_uabal>;
+ def : Pat<(nxv8i16 (partial_reduce_umla nxv8i16:$Acc, (AArch64sabd_p (SVEAnyPredicate), nxv16i8:$Op1, nxv16i8:$Op2), (nxv16i8 (splat_vector (i32 1))))),
+ (SABAL_ZZZ_BtoH $Acc, $Op1, $Op2)>;
+ def : Pat<(nxv4i32 (partial_reduce_umla nxv4i32:$Acc, (AArch64sabd_p (SVEAnyPredicate), nxv8i16:$Op1, nxv8i16:$Op2), (nxv8i16 (splat_vector (i32 1))))),
+ (SABAL_ZZZ_HtoS $Acc, $Op1, $Op2)>;
+ def : Pat<(nxv2i64 (partial_reduce_umla nxv2i64:$Acc, (AArch64sabd_p (SVEAnyPredicate), nxv4i32:$Op1, nxv4i32:$Op2), (nxv4i32 (splat_vector (i32 1))))),
+ (SABAL_ZZZ_StoD $Acc, $Op1, $Op2)>;
+
+ def : Pat<(nxv8i16 (partial_reduce_umla nxv8i16:$Acc, (AArch64uabd_p (SVEAnyPredicate), nxv16i8:$Op1, nxv16i8:$Op2), (nxv16i8 (splat_vector (i32 1))))),
+ (UABAL_ZZZ_BtoH $Acc, $Op1, $Op2)>;
+ def : Pat<(nxv4i32 (partial_reduce_umla nxv4i32:$Acc, (AArch64uabd_p (SVEAnyPredicate), nxv8i16:$Op1, nxv8i16:$Op2), (nxv8i16 (splat_vector (i32 1))))),
+ (UABAL_ZZZ_HtoS $Acc, $Op1, $Op2)>;
+ def : Pat<(nxv2i64 (partial_reduce_umla nxv2i64:$Acc, (AArch64uabd_p (SVEAnyPredicate), nxv4i32:$Op1, nxv4i32:$Op2), (nxv4i32 (splat_vector (i32 1))))),
+ (UABAL_ZZZ_StoD $Acc, $Op1, $Op2)>;
+
// SVE2 integer dot product
defm SDOT_ZZZ_BtoH : sve2p3_two_way_dot_vv<"sdot", 0b0, int_aarch64_sve_sdot_x2>;
defm UDOT_ZZZ_BtoH : sve2p3_two_way_dot_vv<"udot", 0b1, int_aarch64_sve_udot_x2>;
diff --git a/llvm/test/CodeGen/AArch64/vector-absolute-difference.ll b/llvm/test/CodeGen/AArch64/vector-absolute-difference.ll
index bebf1cb00e6a9..99ec1710c3687 100644
--- a/llvm/test/CodeGen/AArch64/vector-absolute-difference.ll
+++ b/llvm/test/CodeGen/AArch64/vector-absolute-difference.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mattr=+sve2,+dotprod < %s | FileCheck %s
+; RUN: llc -mattr=+sve2,+dotprod < %s | FileCheck %s --check-prefixes=CHECK,SVE2
+; RUN: llc -mattr=+sve2p3,+dotprod < %s | FileCheck %s --check-prefixes=CHECK,SVE2p3
target triple = "aarch64-unknown-linux-gnu"
@@ -27,21 +28,61 @@ define <vscale x 16 x i8> @uabs_nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8
ret <vscale x 16 x i8> %uabs
}
-; TODO: This case could be lowered to a sabal[bt] pair.
define <vscale x 8 x i16> @sabs_nxv16i8_wide_add(<vscale x 8 x i16> %acc, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
-; CHECK-LABEL: sabs_nxv16i8_wide_add:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: sabd z1.b, p0/m, z1.b, z2.b
-; CHECK-NEXT: uaddwb z0.h, z0.h, z1.b
-; CHECK-NEXT: uaddwt z0.h, z0.h, z1.b
-; CHECK-NEXT: ret
- %smax = tail call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
- %smin = tail call <vscale x 16 x i8> @llvm.smin.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; SVE2-LABEL: sabs_nxv16i8_wide_add:
+; SVE2: // %bb.0:
+; SVE2-NEXT: sabalb z0.h, z1.b, z2.b
+; SVE2-NEXT: sabalt z0.h, z1.b, z2.b
+; SVE2-NEXT: ret
+;
+; SVE2p3-LABEL: sabs_nxv16i8_wide_add:
+; SVE2p3: // %bb.0:
+; SVE2p3-NEXT: sabal z0.h, z1.b, z2.b
+; SVE2p3-NEXT: ret
+ %smax = tail call <vscale x 16 x i8> @llvm.smax(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+ %smin = tail call <vscale x 16 x i8> @llvm.smin(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
%sabs = sub <vscale x 16 x i8> %smax, %smin
%ext = zext <vscale x 16 x i8> %sabs to <vscale x 16 x i16>
- %reduce = call <vscale x 8 x i16> @llvm.vector.partial.reduce.add.v4i32.v16i32(<vscale x 8 x i16> %acc, <vscale x 16 x i16> %ext)
- ret <vscale x 8 x i16> %reduce
+ %reduce = call <vscale x 8 x i16> @llvm.vector.partial.reduce.add(<vscale x 8 x i16> %acc, <vscale x 16 x i16> %ext)
+ ret <vscale x 8 x i16> %reduce
+}
+
+define <vscale x 4 x i32> @sabs_nxv8i16_wide_add(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; SVE2-LABEL: sabs_nxv8i16_wide_add:
+; SVE2: // %bb.0:
+; SVE2-NEXT: sabalb z0.s, z1.h, z2.h
+; SVE2-NEXT: sabalt z0.s, z1.h, z2.h
+; SVE2-NEXT: ret
+;
+; SVE2p3-LABEL: sabs_nxv8i16_wide_add:
+; SVE2p3: // %bb.0:
+; SVE2p3-NEXT: sabal z0.s, z1.h, z2.h
+; SVE2p3-NEXT: ret
+ %smax = tail call <vscale x 8 x i16> @llvm.smax(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+ %smin = tail call <vscale x 8 x i16> @llvm.smin(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+ %sabs = sub <vscale x 8 x i16> %smax, %smin
+ %ext = zext <vscale x 8 x i16> %sabs to <vscale x 8 x i32>
+ %reduce = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add(<vscale x 4 x i32> %acc, <vscale x 8 x i32> %ext)
+ ret <vscale x 4 x i32> %reduce
+}
+
+define <vscale x 2 x i64> @sabs_nxv4i32_wide_add(<vscale x 2 x i64> %acc, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; SVE2-LABEL: sabs_nxv4i32_wide_add:
+; SVE2: // %bb.0:
+; SVE2-NEXT: sabalb z0.d, z1.s, z2.s
+; SVE2-NEXT: sabalt z0.d, z1.s, z2.s
+; SVE2-NEXT: ret
+;
+; SVE2p3-LABEL: sabs_nxv4i32_wide_add:
+; SVE2p3: // %bb.0:
+; SVE2p3-NEXT: sabal z0.d, z1.s, z2.s
+; SVE2p3-NEXT: ret
+ %smax = tail call <vscale x 4 x i32> @llvm.smax(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
+ %smin = tail call <vscale x 4 x i32> @llvm.smin(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
+ %sabs = sub <vscale x 4 x i32> %smax, %smin
+ %ext = zext <vscale x 4 x i32> %sabs to <vscale x 4 x i64>
+ %reduce = call <vscale x 2 x i64> @llvm.vector.partial.reduce.add(<vscale x 2 x i64> %acc, <vscale x 4 x i64> %ext)
+ ret <vscale x 2 x i64> %reduce
}
define <vscale x 4 x i32> @sabs_nxv16i8_dot(<vscale x 4 x i32> %acc, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
@@ -82,23 +123,63 @@ define <16 x i8> @uabs_v16i8(<16 x i8> %a, <16 x i8> %b) {
ret <16 x i8> %uabs
}
-; TODO: This case could be lowered to a uabal[bt] pair.
-define <vscale x 4 x i32> @uabs_nxv16i8_wide_add(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
-; CHECK-LABEL: uabs_nxv16i8_wide_add:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.h
-; CHECK-NEXT: uabd z1.h, p0/m, z1.h, z2.h
-; CHECK-NEXT: uaddwb z0.s, z0.s, z1.h
-; CHECK-NEXT: uaddwt z0.s, z0.s, z1.h
-; CHECK-NEXT: ret
- %umax = tail call <vscale x 8 x i16> @llvm.umax.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
- %umin = tail call <vscale x 8 x i16> @llvm.umin.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+define <vscale x 8 x i16> @uabs_nxv16i8_wide_add(<vscale x 8 x i16> %acc, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; SVE2-LABEL: uabs_nxv16i8_wide_add:
+; SVE2: // %bb.0:
+; SVE2-NEXT: uabalb z0.h, z1.b, z2.b
+; SVE2-NEXT: uabalt z0.h, z1.b, z2.b
+; SVE2-NEXT: ret
+;
+; SVE2p3-LABEL: uabs_nxv16i8_wide_add:
+; SVE2p3: // %bb.0:
+; SVE2p3-NEXT: uabal z0.h, z1.b, z2.b
+; SVE2p3-NEXT: ret
+ %umax = tail call <vscale x 16 x i8> @llvm.umax(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+ %umin = tail call <vscale x 16 x i8> @llvm.umin(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+ %uabs = sub <vscale x 16 x i8> %umax, %umin
+ %ext = zext <vscale x 16 x i8> %uabs to <vscale x 16 x i16>
+ %reduce = call <vscale x 8 x i16> @llvm.vector.partial.reduce.add(<vscale x 8 x i16> %acc, <vscale x 16 x i16> %ext)
+ ret <vscale x 8 x i16> %reduce
+}
+
+define <vscale x 4 x i32> @uabs_nxv8i16_wide_add(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; SVE2-LABEL: uabs_nxv8i16_wide_add:
+; SVE2: // %bb.0:
+; SVE2-NEXT: uabalb z0.s, z1.h, z2.h
+; SVE2-NEXT: uabalt z0.s, z1.h, z2.h
+; SVE2-NEXT: ret
+;
+; SVE2p3-LABEL: uabs_nxv8i16_wide_add:
+; SVE2p3: // %bb.0:
+; SVE2p3-NEXT: uabal z0.s, z1.h, z2.h
+; SVE2p3-NEXT: ret
+ %umax = tail call <vscale x 8 x i16> @llvm.umax(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+ %umin = tail call <vscale x 8 x i16> @llvm.umin(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
%uabs = sub <vscale x 8 x i16> %umax, %umin
%ext = zext <vscale x 8 x i16> %uabs to <vscale x 8 x i32>
- %reduce = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<vscale x 4 x i32> %acc, <vscale x 8 x i32> %ext)
+ %reduce = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add(<vscale x 4 x i32> %acc, <vscale x 8 x i32> %ext)
ret <vscale x 4 x i32> %reduce
}
+define <vscale x 2 x i64> @uabs_nxv4i32_wide_add(<vscale x 2 x i64> %acc, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; SVE2-LABEL: uabs_nxv4i32_wide_add:
+; SVE2: // %bb.0:
+; SVE2-NEXT: uabalb z0.d, z1.s, z2.s
+; SVE2-NEXT: uabalt z0.d, z1.s, z2.s
+; SVE2-NEXT: ret
+;
+; SVE2p3-LABEL: uabs_nxv4i32_wide_add:
+; SVE2p3: // %bb.0:
+; SVE2p3-NEXT: uabal z0.d, z1.s, z2.s
+; SVE2p3-NEXT: ret
+ %umax = tail call <vscale x 4 x i32> @llvm.umax(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
+ %umin = tail call <vscale x 4 x i32> @llvm.umin(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
+ %uabs = sub <vscale x 4 x i32> %umax, %umin
+ %ext = zext <vscale x 4 x i32> %uabs to <vscale x 4 x i64>
+ %reduce = call <vscale x 2 x i64> @llvm.vector.partial.reduce.add(<vscale x 2 x i64> %acc, <vscale x 4 x i64> %ext)
+ ret <vscale x 2 x i64> %reduce
+}
+
define <4 x i32> @uabs_v16i8_dot(<4 x i32> %acc, <16 x i8> %a, <16 x i8> %b) {
; CHECK-LABEL: uabs_v16i8_dot:
; CHECK: // %bb.0:
``````````
</details>
https://github.com/llvm/llvm-project/pull/212800
More information about the llvm-commits
mailing list