[llvm] a60571c - [AArch64] Select [SU]ADDLP for a partial reduction into a zero accumulator (#217186)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 20 03:23:59 PDT 2026
Author: Adam Scott
Date: 2026-08-20T11:23:54+01:00
New Revision: a60571cd2e9f981216e772cdc40954987a1e1dfa
URL: https://github.com/llvm/llvm-project/commit/a60571cd2e9f981216e772cdc40954987a1e1dfa
DIFF: https://github.com/llvm/llvm-project/commit/a60571cd2e9f981216e772cdc40954987a1e1dfa.diff
LOG: [AArch64] Select [SU]ADDLP for a partial reduction into a zero accumulator (#217186)
A partial add reduction into a zero accumulator still materialises the
zero because [SU]ADALP needs something to add to. Select [SU]ADDLP
instead, which is the same pairwise widening add with no accumulator.
```llvm
%wide = zext <16 x i8> %a to <16 x i16>
%r = call <8 x i16> @llvm.vector.partial.reduce.add(<8 x i16> zeroinitializer, <16 x i16> %wide)
```
before:
```
movi v1.2d, #0000000000000000
uadalp v1.8h, v0.16b
mov v0.16b, v1.16b
```
after:
```
uaddlp v0.8h, v0.16b
```
Split out of #214636
Added:
Modified:
llvm/lib/Target/AArch64/AArch64InstrInfo.td
llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index da6b90e9636f8..44e649484b8a2 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -6085,6 +6085,15 @@ multiclass SelectVectorPartialReduceAdd<ValueType DstVT, ValueType SrcVT, dag im
def : Pat<(DstVT (partial_reduce_umla DstVT:$Acc, SrcVT:$Input, (SrcVT immOneV))),
(!cast<Instruction>("UADALP" # SrcVT # "_" # DstVT) $Acc, $Input)>;
+
+ // A zero accumulator has nothing to add to so it lowers to [SU]ADDLP.
+ def : Pat<(DstVT (partial_reduce_smla (DstVT immAllZerosV), SrcVT:$Input,
+ (SrcVT immOneV))),
+ (!cast<Instruction>("SADDLP" # SrcVT # "_" # DstVT) $Input)>;
+
+ def : Pat<(DstVT (partial_reduce_umla (DstVT immAllZerosV), SrcVT:$Input,
+ (SrcVT immOneV))),
+ (!cast<Instruction>("UADDLP" # SrcVT # "_" # DstVT) $Input)>;
}
defm : SelectVectorPartialReduceAdd<v4i16, v8i8, (AArch64movi (i32 1))>;
diff --git a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
index 171253581fbe6..3d84807b06995 100644
--- a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
+++ b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
@@ -1771,3 +1771,152 @@ define <2 x i64> @partial_reduce_sext_cmp_i32tov2i64(<2 x i64> %acc, <4 x i32> %
ret <2 x i64> %partial.reduce
}
+
+define <4 x i16> @partial_reduce_zext_zeroacc_v8i8tov4i16(<8 x i8> %a) {
+; CHECK-COMMON-LABEL: partial_reduce_zext_zeroacc_v8i8tov4i16:
+; CHECK-COMMON: // %bb.0:
+; CHECK-COMMON-NEXT: uaddlp v0.4h, v0.8b
+; CHECK-COMMON-NEXT: ret
+ %a.wide = zext <8 x i8> %a to <8 x i16>
+ %partial.reduce = tail call <4 x i16> @llvm.vector.partial.reduce.add(<4 x i16> zeroinitializer, <8 x i16> %a.wide)
+ ret <4 x i16> %partial.reduce
+}
+
+define <4 x i16> @partial_reduce_sext_zeroacc_v8i8tov4i16(<8 x i8> %a) {
+; CHECK-COMMON-LABEL: partial_reduce_sext_zeroacc_v8i8tov4i16:
+; CHECK-COMMON: // %bb.0:
+; CHECK-COMMON-NEXT: saddlp v0.4h, v0.8b
+; CHECK-COMMON-NEXT: ret
+ %a.wide = sext <8 x i8> %a to <8 x i16>
+ %partial.reduce = tail call <4 x i16> @llvm.vector.partial.reduce.add(<4 x i16> zeroinitializer, <8 x i16> %a.wide)
+ ret <4 x i16> %partial.reduce
+}
+
+define <8 x i16> @partial_reduce_zext_zeroacc_v16i8tov8i16(<16 x i8> %a) {
+; CHECK-COMMON-LABEL: partial_reduce_zext_zeroacc_v16i8tov8i16:
+; CHECK-COMMON: // %bb.0:
+; CHECK-COMMON-NEXT: uaddlp v0.8h, v0.16b
+; CHECK-COMMON-NEXT: ret
+ %a.wide = zext <16 x i8> %a to <16 x i16>
+ %partial.reduce = tail call <8 x i16> @llvm.vector.partial.reduce.add(<8 x i16> zeroinitializer, <16 x i16> %a.wide)
+ ret <8 x i16> %partial.reduce
+}
+
+define <8 x i16> @partial_reduce_sext_zeroacc_v16i8tov8i16(<16 x i8> %a) {
+; CHECK-COMMON-LABEL: partial_reduce_sext_zeroacc_v16i8tov8i16:
+; CHECK-COMMON: // %bb.0:
+; CHECK-COMMON-NEXT: saddlp v0.8h, v0.16b
+; CHECK-COMMON-NEXT: ret
+ %a.wide = sext <16 x i8> %a to <16 x i16>
+ %partial.reduce = tail call <8 x i16> @llvm.vector.partial.reduce.add(<8 x i16> zeroinitializer, <16 x i16> %a.wide)
+ ret <8 x i16> %partial.reduce
+}
+
+define <2 x i32> @partial_reduce_zext_zeroacc_v4i16tov2i32(<4 x i16> %a) {
+; CHECK-COMMON-LABEL: partial_reduce_zext_zeroacc_v4i16tov2i32:
+; CHECK-COMMON: // %bb.0:
+; CHECK-COMMON-NEXT: uaddlp v0.2s, v0.4h
+; CHECK-COMMON-NEXT: ret
+ %a.wide = zext <4 x i16> %a to <4 x i32>
+ %partial.reduce = tail call <2 x i32> @llvm.vector.partial.reduce.add(<2 x i32> zeroinitializer, <4 x i32> %a.wide)
+ ret <2 x i32> %partial.reduce
+}
+
+define <2 x i32> @partial_reduce_sext_zeroacc_v4i16tov2i32(<4 x i16> %a) {
+; CHECK-COMMON-LABEL: partial_reduce_sext_zeroacc_v4i16tov2i32:
+; CHECK-COMMON: // %bb.0:
+; CHECK-COMMON-NEXT: saddlp v0.2s, v0.4h
+; CHECK-COMMON-NEXT: ret
+ %a.wide = sext <4 x i16> %a to <4 x i32>
+ %partial.reduce = tail call <2 x i32> @llvm.vector.partial.reduce.add(<2 x i32> zeroinitializer, <4 x i32> %a.wide)
+ ret <2 x i32> %partial.reduce
+}
+
+define <4 x i32> @partial_reduce_zext_zeroacc_v8i16tov4i32(<8 x i16> %a) {
+; CHECK-COMMON-LABEL: partial_reduce_zext_zeroacc_v8i16tov4i32:
+; CHECK-COMMON: // %bb.0:
+; CHECK-COMMON-NEXT: uaddlp v0.4s, v0.8h
+; CHECK-COMMON-NEXT: ret
+ %a.wide = zext <8 x i16> %a to <8 x i32>
+ %partial.reduce = tail call <4 x i32> @llvm.vector.partial.reduce.add(<4 x i32> zeroinitializer, <8 x i32> %a.wide)
+ ret <4 x i32> %partial.reduce
+}
+
+define <4 x i32> @partial_reduce_sext_zeroacc_v8i16tov4i32(<8 x i16> %a) {
+; CHECK-COMMON-LABEL: partial_reduce_sext_zeroacc_v8i16tov4i32:
+; CHECK-COMMON: // %bb.0:
+; CHECK-COMMON-NEXT: saddlp v0.4s, v0.8h
+; CHECK-COMMON-NEXT: ret
+ %a.wide = sext <8 x i16> %a to <8 x i32>
+ %partial.reduce = tail call <4 x i32> @llvm.vector.partial.reduce.add(<4 x i32> zeroinitializer, <8 x i32> %a.wide)
+ ret <4 x i32> %partial.reduce
+}
+
+define <1 x i64> @partial_reduce_zext_zeroacc_v2i32tov1i64(<2 x i32> %a) {
+; CHECK-COMMON-LABEL: partial_reduce_zext_zeroacc_v2i32tov1i64:
+; CHECK-COMMON: // %bb.0:
+; CHECK-COMMON-NEXT: uaddlp v0.1d, v0.2s
+; CHECK-COMMON-NEXT: ret
+ %a.wide = zext <2 x i32> %a to <2 x i64>
+ %partial.reduce = tail call <1 x i64> @llvm.vector.partial.reduce.add(<1 x i64> zeroinitializer, <2 x i64> %a.wide)
+ ret <1 x i64> %partial.reduce
+}
+
+define <1 x i64> @partial_reduce_sext_zeroacc_v2i32tov1i64(<2 x i32> %a) {
+; CHECK-COMMON-LABEL: partial_reduce_sext_zeroacc_v2i32tov1i64:
+; CHECK-COMMON: // %bb.0:
+; CHECK-COMMON-NEXT: saddlp v0.1d, v0.2s
+; CHECK-COMMON-NEXT: ret
+ %a.wide = sext <2 x i32> %a to <2 x i64>
+ %partial.reduce = tail call <1 x i64> @llvm.vector.partial.reduce.add(<1 x i64> zeroinitializer, <2 x i64> %a.wide)
+ ret <1 x i64> %partial.reduce
+}
+
+define <2 x i64> @partial_reduce_zext_zeroacc_v4i32tov2i64(<4 x i32> %a) {
+; CHECK-COMMON-LABEL: partial_reduce_zext_zeroacc_v4i32tov2i64:
+; CHECK-COMMON: // %bb.0:
+; CHECK-COMMON-NEXT: uaddlp v0.2d, v0.4s
+; CHECK-COMMON-NEXT: ret
+ %a.wide = zext <4 x i32> %a to <4 x i64>
+ %partial.reduce = tail call <2 x i64> @llvm.vector.partial.reduce.add(<2 x i64> zeroinitializer, <4 x i64> %a.wide)
+ ret <2 x i64> %partial.reduce
+}
+
+define <2 x i64> @partial_reduce_sext_zeroacc_v4i32tov2i64(<4 x i32> %a) {
+; CHECK-COMMON-LABEL: partial_reduce_sext_zeroacc_v4i32tov2i64:
+; CHECK-COMMON: // %bb.0:
+; CHECK-COMMON-NEXT: saddlp v0.2d, v0.4s
+; CHECK-COMMON-NEXT: ret
+ %a.wide = sext <4 x i32> %a to <4 x i64>
+ %partial.reduce = tail call <2 x i64> @llvm.vector.partial.reduce.add(<2 x i64> zeroinitializer, <4 x i64> %a.wide)
+ ret <2 x i64> %partial.reduce
+}
+
+define <4 x i32> @partial_reduce_umull_zeroacc_v16i8tov4i32(<16 x i8> %u, <16 x i8> %s) {
+; CHECK-NODOT-LABEL: partial_reduce_umull_zeroacc_v16i8tov4i32:
+; CHECK-NODOT: // %bb.0:
+; CHECK-NODOT-NEXT: umull v2.8h, v1.8b, v0.8b
+; CHECK-NODOT-NEXT: umull2 v1.8h, v1.16b, v0.16b
+; CHECK-NODOT-NEXT: uaddlp v0.4s, v2.8h
+; CHECK-NODOT-NEXT: uadalp v0.4s, v1.8h
+; CHECK-NODOT-NEXT: ret
+;
+; CHECK-DOT-LABEL: partial_reduce_umull_zeroacc_v16i8tov4i32:
+; CHECK-DOT: // %bb.0:
+; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000
+; CHECK-DOT-NEXT: udot v2.4s, v1.16b, v0.16b
+; CHECK-DOT-NEXT: mov v0.16b, v2.16b
+; CHECK-DOT-NEXT: ret
+;
+; CHECK-DOT-I8MM-LABEL: partial_reduce_umull_zeroacc_v16i8tov4i32:
+; CHECK-DOT-I8MM: // %bb.0:
+; CHECK-DOT-I8MM-NEXT: movi v2.2d, #0000000000000000
+; CHECK-DOT-I8MM-NEXT: udot v2.4s, v1.16b, v0.16b
+; CHECK-DOT-I8MM-NEXT: mov v0.16b, v2.16b
+; CHECK-DOT-I8MM-NEXT: ret
+ %u.wide = zext <16 x i8> %u to <16 x i32>
+ %s.wide = zext <16 x i8> %s to <16 x i32>
+ %mult = mul nuw nsw <16 x i32> %s.wide, %u.wide
+ %partial.reduce = tail call <4 x i32> @llvm.vector.partial.reduce.add(<4 x i32> zeroinitializer, <16 x i32> %mult)
+ ret <4 x i32> %partial.reduce
+}
More information about the llvm-commits
mailing list