[llvm] [AArch64] Add SVE patterns for [su]adalp with separate ADD. (PR #204796)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 19 03:52:56 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Gaëtan Bossu (gbossu)
<details>
<summary>Changes</summary>
It can happen that the [su]adalp uses a zero accumulator and its result is then an operand into an ADD instruction. We can fold those.
---
Full diff: https://github.com/llvm/llvm-project/pull/204796.diff
2 Files Affected:
- (modified) llvm/lib/Target/AArch64/SVEInstrFormats.td (+7)
- (modified) llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-pairwise-arith.ll (+85)
``````````diff
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index f96702a01c277..f916fe71c02ae 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -4223,6 +4223,13 @@ multiclass sve2_int_sadd_long_accum_pairwise<bit U, string asm, SDPatternOperato
def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>;
def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>;
def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>;
+
+ def : Pat<(nxv8i16 (add nxv8i16:$Op2, (nxv8i16 (op nxv8i1:$Op1, (nxv8i16 (SVEDup0)), nxv16i8:$Op3)))),
+ (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3)>;
+ def : Pat<(nxv4i32 (add nxv4i32:$Op2, (nxv4i32 (op nxv4i1:$Op1, (nxv4i32 (SVEDup0)), nxv8i16:$Op3)))),
+ (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3)>;
+ def : Pat<(nxv2i64 (add nxv2i64:$Op2, (nxv2i64 (op nxv2i1:$Op1, (nxv2i64 (SVEDup0)), nxv4i32:$Op3)))),
+ (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3)>;
}
class sve2_int_un_pred_arit<bits<2> sz, bits<2> opc,
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-pairwise-arith.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-pairwise-arith.ll
index 7e852f7f1c287..9a1388a9bee04 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-pairwise-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-pairwise-arith.ll
@@ -40,6 +40,55 @@ define <vscale x 2 x i64> @sadalp_i32(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
ret <vscale x 2 x i64> %out
}
+define <vscale x 8 x i16> @sadalp_i8_separate_add(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: sadalp_i8_separate_add:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sadalp z0.h, p0/m, z1.b
+; CHECK-NEXT: ret
+ %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sadalp.nxv8i16(<vscale x 8 x i1> %pg,
+ <vscale x 8 x i16> zeroinitializer,
+ <vscale x 16 x i8> %b)
+ %out2 = add <vscale x 8 x i16> %out, %a
+ ret <vscale x 8 x i16> %out2
+}
+
+define <vscale x 4 x i32> @sadalp_i16_separate_add(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: sadalp_i16_separate_add:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sadalp z0.s, p0/m, z1.h
+; CHECK-NEXT: ret
+ %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sadalp.nxv4i32(<vscale x 4 x i1> %pg,
+ <vscale x 4 x i32> zeroinitializer,
+ <vscale x 8 x i16> %b)
+ %out2 = add <vscale x 4 x i32> %out, %a
+ ret <vscale x 4 x i32> %out2
+}
+
+define <vscale x 2 x i64> @sadalp_i32_separate_add(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sadalp_i32_separate_add:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sadalp z0.d, p0/m, z1.s
+; CHECK-NEXT: ret
+ %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sadalp.nxv2i64(<vscale x 2 x i1> %pg,
+ <vscale x 2 x i64> zeroinitializer,
+ <vscale x 4 x i32> %b)
+ %out2 = add <vscale x 2 x i64> %out, %a
+ ret <vscale x 2 x i64> %out2
+}
+
+; Make sure the pattern is commutative
+define <vscale x 2 x i64> @sadalp_i32_separate_add_rev(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sadalp_i32_separate_add_rev:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sadalp z0.d, p0/m, z1.s
+; CHECK-NEXT: ret
+ %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sadalp.nxv2i64(<vscale x 2 x i1> %pg,
+ <vscale x 2 x i64> zeroinitializer,
+ <vscale x 4 x i32> %b)
+ %out2 = add <vscale x 2 x i64> %a, %out
+ ret <vscale x 2 x i64> %out2
+}
+
;
; UADALP
;
@@ -77,6 +126,42 @@ define <vscale x 2 x i64> @uadalp_i32(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
ret <vscale x 2 x i64> %out
}
+define <vscale x 8 x i16> @uadalp_i8_separate_add(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: uadalp_i8_separate_add:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uadalp z0.h, p0/m, z1.b
+; CHECK-NEXT: ret
+ %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uadalp.nxv8i16(<vscale x 8 x i1> %pg,
+ <vscale x 8 x i16> zeroinitializer,
+ <vscale x 16 x i8> %b)
+ %out2 = add <vscale x 8 x i16> %out, %a
+ ret <vscale x 8 x i16> %out2
+}
+
+define <vscale x 4 x i32> @uadalp_i16_separate_add(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: uadalp_i16_separate_add:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uadalp z0.s, p0/m, z1.h
+; CHECK-NEXT: ret
+ %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uadalp.nxv4i32(<vscale x 4 x i1> %pg,
+ <vscale x 4 x i32> zeroinitializer,
+ <vscale x 8 x i16> %b)
+ %out2 = add <vscale x 4 x i32> %out, %a
+ ret <vscale x 4 x i32> %out2
+}
+
+define <vscale x 2 x i64> @uadalp_i32_separate_add(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: uadalp_i32_separate_add:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uadalp z0.d, p0/m, z1.s
+; CHECK-NEXT: ret
+ %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uadalp.nxv2i64(<vscale x 2 x i1> %pg,
+ <vscale x 2 x i64> zeroinitializer,
+ <vscale x 4 x i32> %b)
+ %out2 = add <vscale x 2 x i64> %out, %a
+ ret <vscale x 2 x i64> %out2
+}
+
declare <vscale x 8 x i16> @llvm.aarch64.sve.sadalp.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 16 x i8>)
declare <vscale x 4 x i32> @llvm.aarch64.sve.sadalp.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 8 x i16>)
declare <vscale x 2 x i64> @llvm.aarch64.sve.sadalp.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 4 x i32>)
``````````
</details>
https://github.com/llvm/llvm-project/pull/204796
More information about the llvm-commits
mailing list