[llvm] f5346a5 - [LLVM][CodeGen][SVE] Add isel patterns for abs-diff partial reductions. (#212800)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 4 06:32:15 PDT 2026
Author: Paul Walker
Date: 2026-08-04T14:32:10+01:00
New Revision: f5346a5a341d09f279063e239f4215478e0f5af3
URL: https://github.com/llvm/llvm-project/commit/f5346a5a341d09f279063e239f4215478e0f5af3
DIFF: https://github.com/llvm/llvm-project/commit/f5346a5a341d09f279063e239f4215478e0f5af3.diff
LOG: [LLVM][CodeGen][SVE] Add isel patterns for abs-diff partial reductions. (#212800)
For SVE2 we can use top/bottom [S/U]ABAL[T/B] instruction. SVE2p3 can
use just [S/U]ABAL.
Added:
Modified:
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
llvm/test/CodeGen/AArch64/vector-absolute-difference.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 55a3db86940be..f4eb8c285e80c 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4215,6 +4215,20 @@ let Predicates = [HasSVE2_or_SME] in {
defm UABALB_ZZZ : sve2_int_abs
diff _accum_long<0b10, "uabalb", int_aarch64_sve_uabalb>;
defm UABALT_ZZZ : sve2_int_abs
diff _accum_long<0b11, "uabalt", int_aarch64_sve_uabalt>;
+ def : Pat<(nxv8i16 (partial_reduce_umla nxv8i16:$Acc, (AArch64sabd_p (SVEAnyPredicate), nxv16i8:$Op1, nxv16i8:$Op2), (nxv16i8 (splat_vector (i32 1))))),
+ (SABALT_ZZZ_H (SABALB_ZZZ_H $Acc, $Op1, $Op2), $Op1, $Op2)>;
+ def : Pat<(nxv4i32 (partial_reduce_umla nxv4i32:$Acc, (AArch64sabd_p (SVEAnyPredicate), nxv8i16:$Op1, nxv8i16:$Op2), (nxv8i16 (splat_vector (i32 1))))),
+ (SABALT_ZZZ_S (SABALB_ZZZ_S $Acc, $Op1, $Op2), $Op1, $Op2)>;
+ def : Pat<(nxv2i64 (partial_reduce_umla nxv2i64:$Acc, (AArch64sabd_p (SVEAnyPredicate), nxv4i32:$Op1, nxv4i32:$Op2), (nxv4i32 (splat_vector (i32 1))))),
+ (SABALT_ZZZ_D (SABALB_ZZZ_D $Acc, $Op1, $Op2), $Op1, $Op2)>;
+
+ def : Pat<(nxv8i16 (partial_reduce_umla nxv8i16:$Acc, (AArch64uabd_p (SVEAnyPredicate), nxv16i8:$Op1, nxv16i8:$Op2), (nxv16i8 (splat_vector (i32 1))))),
+ (UABALT_ZZZ_H (UABALB_ZZZ_H $Acc, $Op1, $Op2), $Op1, $Op2)>;
+ def : Pat<(nxv4i32 (partial_reduce_umla nxv4i32:$Acc, (AArch64uabd_p (SVEAnyPredicate), nxv8i16:$Op1, nxv8i16:$Op2), (nxv8i16 (splat_vector (i32 1))))),
+ (UABALT_ZZZ_S (UABALB_ZZZ_S $Acc, $Op1, $Op2), $Op1, $Op2)>;
+ def : Pat<(nxv2i64 (partial_reduce_umla nxv2i64:$Acc, (AArch64uabd_p (SVEAnyPredicate), nxv4i32:$Op1, nxv4i32:$Op2), (nxv4i32 (splat_vector (i32 1))))),
+ (UABALT_ZZZ_D (UABALB_ZZZ_D $Acc, $Op1, $Op2), $Op1, $Op2)>;
+
// SVE2 integer add/subtract long with carry
defm ADCLB_ZZZ : sve2_int_addsub_long_carry<0b00, "adclb", int_aarch64_sve_adclb>;
defm ADCLT_ZZZ : sve2_int_addsub_long_carry<0b01, "adclt", int_aarch64_sve_adclt>;
@@ -5076,6 +5090,20 @@ let Predicates = [HasSVE2p3_or_SME2p3] in {
defm SABAL_ZZZ : sve2_int_two_way_abs
diff _accum_long<0b0, "sabal", int_aarch64_sve_sabal>;
defm UABAL_ZZZ : sve2_int_two_way_abs
diff _accum_long<0b1, "uabal", int_aarch64_sve_uabal>;
+ def : Pat<(nxv8i16 (partial_reduce_umla nxv8i16:$Acc, (AArch64sabd_p (SVEAnyPredicate), nxv16i8:$Op1, nxv16i8:$Op2), (nxv16i8 (splat_vector (i32 1))))),
+ (SABAL_ZZZ_BtoH $Acc, $Op1, $Op2)>;
+ def : Pat<(nxv4i32 (partial_reduce_umla nxv4i32:$Acc, (AArch64sabd_p (SVEAnyPredicate), nxv8i16:$Op1, nxv8i16:$Op2), (nxv8i16 (splat_vector (i32 1))))),
+ (SABAL_ZZZ_HtoS $Acc, $Op1, $Op2)>;
+ def : Pat<(nxv2i64 (partial_reduce_umla nxv2i64:$Acc, (AArch64sabd_p (SVEAnyPredicate), nxv4i32:$Op1, nxv4i32:$Op2), (nxv4i32 (splat_vector (i32 1))))),
+ (SABAL_ZZZ_StoD $Acc, $Op1, $Op2)>;
+
+ def : Pat<(nxv8i16 (partial_reduce_umla nxv8i16:$Acc, (AArch64uabd_p (SVEAnyPredicate), nxv16i8:$Op1, nxv16i8:$Op2), (nxv16i8 (splat_vector (i32 1))))),
+ (UABAL_ZZZ_BtoH $Acc, $Op1, $Op2)>;
+ def : Pat<(nxv4i32 (partial_reduce_umla nxv4i32:$Acc, (AArch64uabd_p (SVEAnyPredicate), nxv8i16:$Op1, nxv8i16:$Op2), (nxv8i16 (splat_vector (i32 1))))),
+ (UABAL_ZZZ_HtoS $Acc, $Op1, $Op2)>;
+ def : Pat<(nxv2i64 (partial_reduce_umla nxv2i64:$Acc, (AArch64uabd_p (SVEAnyPredicate), nxv4i32:$Op1, nxv4i32:$Op2), (nxv4i32 (splat_vector (i32 1))))),
+ (UABAL_ZZZ_StoD $Acc, $Op1, $Op2)>;
+
// SVE2 integer dot product
defm SDOT_ZZZ_BtoH : sve2p3_two_way_dot_vv<"sdot", 0b0, int_aarch64_sve_sdot_x2>;
defm UDOT_ZZZ_BtoH : sve2p3_two_way_dot_vv<"udot", 0b1, int_aarch64_sve_udot_x2>;
diff --git a/llvm/test/CodeGen/AArch64/vector-absolute-
diff erence.ll b/llvm/test/CodeGen/AArch64/vector-absolute-
diff erence.ll
index 45a4b403fe391..99ec1710c3687 100644
--- a/llvm/test/CodeGen/AArch64/vector-absolute-
diff erence.ll
+++ b/llvm/test/CodeGen/AArch64/vector-absolute-
diff erence.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mattr=+sve2,+dotprod < %s | FileCheck %s
+; RUN: llc -mattr=+sve2,+dotprod < %s | FileCheck %s --check-prefixes=CHECK,SVE2
+; RUN: llc -mattr=+sve2p3,+dotprod < %s | FileCheck %s --check-prefixes=CHECK,SVE2p3
target triple = "aarch64-unknown-linux-gnu"
@@ -27,21 +28,61 @@ define <vscale x 16 x i8> @uabs_nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8
ret <vscale x 16 x i8> %uabs
}
-; TODO: This case could be lowered to a sabal[bt] pair.
define <vscale x 8 x i16> @sabs_nxv16i8_wide_add(<vscale x 8 x i16> %acc, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
-; CHECK-LABEL: sabs_nxv16i8_wide_add:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: sabd z1.b, p0/m, z1.b, z2.b
-; CHECK-NEXT: ptrue p0.h
-; CHECK-NEXT: uadalp z0.h, p0/m, z1.b
-; CHECK-NEXT: ret
- %smax = tail call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
- %smin = tail call <vscale x 16 x i8> @llvm.smin.nxv16i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; SVE2-LABEL: sabs_nxv16i8_wide_add:
+; SVE2: // %bb.0:
+; SVE2-NEXT: sabalb z0.h, z1.b, z2.b
+; SVE2-NEXT: sabalt z0.h, z1.b, z2.b
+; SVE2-NEXT: ret
+;
+; SVE2p3-LABEL: sabs_nxv16i8_wide_add:
+; SVE2p3: // %bb.0:
+; SVE2p3-NEXT: sabal z0.h, z1.b, z2.b
+; SVE2p3-NEXT: ret
+ %smax = tail call <vscale x 16 x i8> @llvm.smax(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+ %smin = tail call <vscale x 16 x i8> @llvm.smin(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
%sabs = sub <vscale x 16 x i8> %smax, %smin
%ext = zext <vscale x 16 x i8> %sabs to <vscale x 16 x i16>
- %reduce = call <vscale x 8 x i16> @llvm.vector.partial.reduce.add.v4i32.v16i32(<vscale x 8 x i16> %acc, <vscale x 16 x i16> %ext)
- ret <vscale x 8 x i16> %reduce
+ %reduce = call <vscale x 8 x i16> @llvm.vector.partial.reduce.add(<vscale x 8 x i16> %acc, <vscale x 16 x i16> %ext)
+ ret <vscale x 8 x i16> %reduce
+}
+
+define <vscale x 4 x i32> @sabs_nxv8i16_wide_add(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; SVE2-LABEL: sabs_nxv8i16_wide_add:
+; SVE2: // %bb.0:
+; SVE2-NEXT: sabalb z0.s, z1.h, z2.h
+; SVE2-NEXT: sabalt z0.s, z1.h, z2.h
+; SVE2-NEXT: ret
+;
+; SVE2p3-LABEL: sabs_nxv8i16_wide_add:
+; SVE2p3: // %bb.0:
+; SVE2p3-NEXT: sabal z0.s, z1.h, z2.h
+; SVE2p3-NEXT: ret
+ %smax = tail call <vscale x 8 x i16> @llvm.smax(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+ %smin = tail call <vscale x 8 x i16> @llvm.smin(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+ %sabs = sub <vscale x 8 x i16> %smax, %smin
+ %ext = zext <vscale x 8 x i16> %sabs to <vscale x 8 x i32>
+ %reduce = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add(<vscale x 4 x i32> %acc, <vscale x 8 x i32> %ext)
+ ret <vscale x 4 x i32> %reduce
+}
+
+define <vscale x 2 x i64> @sabs_nxv4i32_wide_add(<vscale x 2 x i64> %acc, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; SVE2-LABEL: sabs_nxv4i32_wide_add:
+; SVE2: // %bb.0:
+; SVE2-NEXT: sabalb z0.d, z1.s, z2.s
+; SVE2-NEXT: sabalt z0.d, z1.s, z2.s
+; SVE2-NEXT: ret
+;
+; SVE2p3-LABEL: sabs_nxv4i32_wide_add:
+; SVE2p3: // %bb.0:
+; SVE2p3-NEXT: sabal z0.d, z1.s, z2.s
+; SVE2p3-NEXT: ret
+ %smax = tail call <vscale x 4 x i32> @llvm.smax(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
+ %smin = tail call <vscale x 4 x i32> @llvm.smin(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
+ %sabs = sub <vscale x 4 x i32> %smax, %smin
+ %ext = zext <vscale x 4 x i32> %sabs to <vscale x 4 x i64>
+ %reduce = call <vscale x 2 x i64> @llvm.vector.partial.reduce.add(<vscale x 2 x i64> %acc, <vscale x 4 x i64> %ext)
+ ret <vscale x 2 x i64> %reduce
}
define <vscale x 4 x i32> @sabs_nxv16i8_dot(<vscale x 4 x i32> %acc, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
@@ -82,23 +123,63 @@ define <16 x i8> @uabs_v16i8(<16 x i8> %a, <16 x i8> %b) {
ret <16 x i8> %uabs
}
-; TODO: This case could be lowered to a uabal[bt] pair.
-define <vscale x 4 x i32> @uabs_nxv16i8_wide_add(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
-; CHECK-LABEL: uabs_nxv16i8_wide_add:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.h
-; CHECK-NEXT: uabd z1.h, p0/m, z1.h, z2.h
-; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: uadalp z0.s, p0/m, z1.h
-; CHECK-NEXT: ret
- %umax = tail call <vscale x 8 x i16> @llvm.umax.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
- %umin = tail call <vscale x 8 x i16> @llvm.umin.nxv8i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+define <vscale x 8 x i16> @uabs_nxv16i8_wide_add(<vscale x 8 x i16> %acc, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
+; SVE2-LABEL: uabs_nxv16i8_wide_add:
+; SVE2: // %bb.0:
+; SVE2-NEXT: uabalb z0.h, z1.b, z2.b
+; SVE2-NEXT: uabalt z0.h, z1.b, z2.b
+; SVE2-NEXT: ret
+;
+; SVE2p3-LABEL: uabs_nxv16i8_wide_add:
+; SVE2p3: // %bb.0:
+; SVE2p3-NEXT: uabal z0.h, z1.b, z2.b
+; SVE2p3-NEXT: ret
+ %umax = tail call <vscale x 16 x i8> @llvm.umax(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+ %umin = tail call <vscale x 16 x i8> @llvm.umin(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+ %uabs = sub <vscale x 16 x i8> %umax, %umin
+ %ext = zext <vscale x 16 x i8> %uabs to <vscale x 16 x i16>
+ %reduce = call <vscale x 8 x i16> @llvm.vector.partial.reduce.add(<vscale x 8 x i16> %acc, <vscale x 16 x i16> %ext)
+ ret <vscale x 8 x i16> %reduce
+}
+
+define <vscale x 4 x i32> @uabs_nxv8i16_wide_add(<vscale x 4 x i32> %acc, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
+; SVE2-LABEL: uabs_nxv8i16_wide_add:
+; SVE2: // %bb.0:
+; SVE2-NEXT: uabalb z0.s, z1.h, z2.h
+; SVE2-NEXT: uabalt z0.s, z1.h, z2.h
+; SVE2-NEXT: ret
+;
+; SVE2p3-LABEL: uabs_nxv8i16_wide_add:
+; SVE2p3: // %bb.0:
+; SVE2p3-NEXT: uabal z0.s, z1.h, z2.h
+; SVE2p3-NEXT: ret
+ %umax = tail call <vscale x 8 x i16> @llvm.umax(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
+ %umin = tail call <vscale x 8 x i16> @llvm.umin(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
%uabs = sub <vscale x 8 x i16> %umax, %umin
%ext = zext <vscale x 8 x i16> %uabs to <vscale x 8 x i32>
- %reduce = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.v4i32.v16i32(<vscale x 4 x i32> %acc, <vscale x 8 x i32> %ext)
+ %reduce = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add(<vscale x 4 x i32> %acc, <vscale x 8 x i32> %ext)
ret <vscale x 4 x i32> %reduce
}
+define <vscale x 2 x i64> @uabs_nxv4i32_wide_add(<vscale x 2 x i64> %acc, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; SVE2-LABEL: uabs_nxv4i32_wide_add:
+; SVE2: // %bb.0:
+; SVE2-NEXT: uabalb z0.d, z1.s, z2.s
+; SVE2-NEXT: uabalt z0.d, z1.s, z2.s
+; SVE2-NEXT: ret
+;
+; SVE2p3-LABEL: uabs_nxv4i32_wide_add:
+; SVE2p3: // %bb.0:
+; SVE2p3-NEXT: uabal z0.d, z1.s, z2.s
+; SVE2p3-NEXT: ret
+ %umax = tail call <vscale x 4 x i32> @llvm.umax(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
+ %umin = tail call <vscale x 4 x i32> @llvm.umin(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
+ %uabs = sub <vscale x 4 x i32> %umax, %umin
+ %ext = zext <vscale x 4 x i32> %uabs to <vscale x 4 x i64>
+ %reduce = call <vscale x 2 x i64> @llvm.vector.partial.reduce.add(<vscale x 2 x i64> %acc, <vscale x 4 x i64> %ext)
+ ret <vscale x 2 x i64> %reduce
+}
+
define <4 x i32> @uabs_v16i8_dot(<4 x i32> %acc, <16 x i8> %a, <16 x i8> %b) {
; CHECK-LABEL: uabs_v16i8_dot:
; CHECK: // %bb.0:
More information about the llvm-commits
mailing list