[llvm] [AArch64] Add SVE patterns for [su]adalp with separate ADD. (PR #204796)
Gaëtan Bossu via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 2 02:13:54 PDT 2026
https://github.com/gbossu updated https://github.com/llvm/llvm-project/pull/204796
>From 747c6941e610b4e3386b9db551907c2e835f6bf0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Fri, 19 Jun 2026 10:46:25 +0000
Subject: [PATCH 1/3] [AArch64] Add SVE patterns for [su]adalp with separate
ADD.
It can happen that the [su]adalp uses a zero accumulater and its result
is then an operand into an ADD instruction. We can fold those.
---
llvm/lib/Target/AArch64/SVEInstrFormats.td | 7 ++
...sve2-intrinsics-widening-pairwise-arith.ll | 85 +++++++++++++++++++
2 files changed, 92 insertions(+)
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index f96702a01c277..f916fe71c02ae 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -4223,6 +4223,13 @@ multiclass sve2_int_sadd_long_accum_pairwise<bit U, string asm, SDPatternOperato
def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>;
def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>;
def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>;
+
+ def : Pat<(nxv8i16 (add nxv8i16:$Op2, (nxv8i16 (op nxv8i1:$Op1, (nxv8i16 (SVEDup0)), nxv16i8:$Op3)))),
+ (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3)>;
+ def : Pat<(nxv4i32 (add nxv4i32:$Op2, (nxv4i32 (op nxv4i1:$Op1, (nxv4i32 (SVEDup0)), nxv8i16:$Op3)))),
+ (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3)>;
+ def : Pat<(nxv2i64 (add nxv2i64:$Op2, (nxv2i64 (op nxv2i1:$Op1, (nxv2i64 (SVEDup0)), nxv4i32:$Op3)))),
+ (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3)>;
}
class sve2_int_un_pred_arit<bits<2> sz, bits<2> opc,
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-pairwise-arith.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-pairwise-arith.ll
index 7e852f7f1c287..9a1388a9bee04 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-pairwise-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-pairwise-arith.ll
@@ -40,6 +40,55 @@ define <vscale x 2 x i64> @sadalp_i32(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
ret <vscale x 2 x i64> %out
}
+define <vscale x 8 x i16> @sadalp_i8_separate_add(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: sadalp_i8_separate_add:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sadalp z0.h, p0/m, z1.b
+; CHECK-NEXT: ret
+ %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sadalp.nxv8i16(<vscale x 8 x i1> %pg,
+ <vscale x 8 x i16> zeroinitializer,
+ <vscale x 16 x i8> %b)
+ %out2 = add <vscale x 8 x i16> %out, %a
+ ret <vscale x 8 x i16> %out2
+}
+
+define <vscale x 4 x i32> @sadalp_i16_separate_add(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: sadalp_i16_separate_add:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sadalp z0.s, p0/m, z1.h
+; CHECK-NEXT: ret
+ %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sadalp.nxv4i32(<vscale x 4 x i1> %pg,
+ <vscale x 4 x i32> zeroinitializer,
+ <vscale x 8 x i16> %b)
+ %out2 = add <vscale x 4 x i32> %out, %a
+ ret <vscale x 4 x i32> %out2
+}
+
+define <vscale x 2 x i64> @sadalp_i32_separate_add(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sadalp_i32_separate_add:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sadalp z0.d, p0/m, z1.s
+; CHECK-NEXT: ret
+ %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sadalp.nxv2i64(<vscale x 2 x i1> %pg,
+ <vscale x 2 x i64> zeroinitializer,
+ <vscale x 4 x i32> %b)
+ %out2 = add <vscale x 2 x i64> %out, %a
+ ret <vscale x 2 x i64> %out2
+}
+
+; Make sure the pattern is commutative
+define <vscale x 2 x i64> @sadalp_i32_separate_add_rev(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: sadalp_i32_separate_add_rev:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sadalp z0.d, p0/m, z1.s
+; CHECK-NEXT: ret
+ %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sadalp.nxv2i64(<vscale x 2 x i1> %pg,
+ <vscale x 2 x i64> zeroinitializer,
+ <vscale x 4 x i32> %b)
+ %out2 = add <vscale x 2 x i64> %a, %out
+ ret <vscale x 2 x i64> %out2
+}
+
;
; UADALP
;
@@ -77,6 +126,42 @@ define <vscale x 2 x i64> @uadalp_i32(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
ret <vscale x 2 x i64> %out
}
+define <vscale x 8 x i16> @uadalp_i8_separate_add(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: uadalp_i8_separate_add:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uadalp z0.h, p0/m, z1.b
+; CHECK-NEXT: ret
+ %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uadalp.nxv8i16(<vscale x 8 x i1> %pg,
+ <vscale x 8 x i16> zeroinitializer,
+ <vscale x 16 x i8> %b)
+ %out2 = add <vscale x 8 x i16> %out, %a
+ ret <vscale x 8 x i16> %out2
+}
+
+define <vscale x 4 x i32> @uadalp_i16_separate_add(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: uadalp_i16_separate_add:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uadalp z0.s, p0/m, z1.h
+; CHECK-NEXT: ret
+ %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uadalp.nxv4i32(<vscale x 4 x i1> %pg,
+ <vscale x 4 x i32> zeroinitializer,
+ <vscale x 8 x i16> %b)
+ %out2 = add <vscale x 4 x i32> %out, %a
+ ret <vscale x 4 x i32> %out2
+}
+
+define <vscale x 2 x i64> @uadalp_i32_separate_add(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: uadalp_i32_separate_add:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uadalp z0.d, p0/m, z1.s
+; CHECK-NEXT: ret
+ %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uadalp.nxv2i64(<vscale x 2 x i1> %pg,
+ <vscale x 2 x i64> zeroinitializer,
+ <vscale x 4 x i32> %b)
+ %out2 = add <vscale x 2 x i64> %out, %a
+ ret <vscale x 2 x i64> %out2
+}
+
declare <vscale x 8 x i16> @llvm.aarch64.sve.sadalp.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 16 x i8>)
declare <vscale x 4 x i32> @llvm.aarch64.sve.sadalp.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 8 x i16>)
declare <vscale x 2 x i64> @llvm.aarch64.sve.sadalp.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 4 x i32>)
>From 47f8e62b24788f7b99b9109c46f2a79fec6a8bae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Tue, 30 Jun 2026 13:24:44 +0000
Subject: [PATCH 2/3] [AArch64][InstCombine] Combine add with [su]adalp
intrinsics
This extends instCombineIntrinsic() to look at aarch64_sve_sadalp and
aarch64_sve_sadalp intrinsics. When those accumulate into a zero
register and they are followed by an add, the latter can be folded in.
---
.../AArch64/AArch64TargetTransformInfo.cpp | 28 ++++++
llvm/lib/Target/AArch64/SVEInstrFormats.td | 7 --
...sve2-intrinsics-widening-pairwise-arith.ll | 85 -----------------
.../AArch64/sve-pairwise-add-long.ll | 94 +++++++++++++++++++
4 files changed, 122 insertions(+), 92 deletions(-)
create mode 100644 llvm/test/Transforms/InstCombine/AArch64/sve-pairwise-add-long.ll
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index f8d2006cbd7de..6f9eab71e6fb6 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -2604,6 +2604,31 @@ instCombineSVEVectorMlaU(InstCombiner &IC, IntrinsicInst &II) {
return std::nullopt;
}
+static std::optional<Instruction *>
+instCombineSVEPairwiseAddLong(InstCombiner &IC, IntrinsicInst &II) {
+ assert((II.getIntrinsicID() == Intrinsic::aarch64_sve_sadalp ||
+ II.getIntrinsicID() == Intrinsic::aarch64_sve_uadalp) &&
+ "Expected SADALP or UADALP intrinsic");
+
+ // We are looking for add(adalp(%pred, zeroinitializer, %in), %acc)
+ if (!II.hasOneUse() || !match(II.getArgOperand(1), m_Zero()))
+ return std::nullopt;
+
+ auto *User = dyn_cast<Instruction>(*II.user_begin());
+ Value *Acc;
+ if (!User || !match(User, m_c_Add(m_Specific(&II), m_Value(Acc))))
+ return std::nullopt;
+
+ IC.Builder.SetInsertPoint(User);
+ Value *PairwiseAddLong = IC.Builder.CreateIntrinsic(
+ II.getIntrinsicID(), {II.getType()},
+ {II.getArgOperand(0), Acc, II.getArgOperand(2)});
+ PairwiseAddLong->takeName(User);
+
+ IC.replaceInstUsesWith(*User, PairwiseAddLong);
+ return IC.eraseInstFromFunction(*User);
+}
+
static std::optional<Instruction *> instCombineSVEVectorAdd(InstCombiner &IC,
IntrinsicInst &II) {
if (auto MLA = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul,
@@ -3171,6 +3196,9 @@ AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC,
IC, II, true);
case Intrinsic::aarch64_sve_mla_u:
return instCombineSVEVectorMlaU(IC, II);
+ case Intrinsic::aarch64_sve_sadalp:
+ case Intrinsic::aarch64_sve_uadalp:
+ return instCombineSVEPairwiseAddLong(IC, II);
case Intrinsic::aarch64_sve_sub:
return instCombineSVEVectorSub(IC, II);
case Intrinsic::aarch64_sve_sub_u:
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index f916fe71c02ae..f96702a01c277 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -4223,13 +4223,6 @@ multiclass sve2_int_sadd_long_accum_pairwise<bit U, string asm, SDPatternOperato
def : SVE_3_Op_Pat<nxv8i16, op, nxv8i1, nxv8i16, nxv16i8, !cast<Instruction>(NAME # _H)>;
def : SVE_3_Op_Pat<nxv4i32, op, nxv4i1, nxv4i32, nxv8i16, !cast<Instruction>(NAME # _S)>;
def : SVE_3_Op_Pat<nxv2i64, op, nxv2i1, nxv2i64, nxv4i32, !cast<Instruction>(NAME # _D)>;
-
- def : Pat<(nxv8i16 (add nxv8i16:$Op2, (nxv8i16 (op nxv8i1:$Op1, (nxv8i16 (SVEDup0)), nxv16i8:$Op3)))),
- (!cast<Instruction>(NAME # _H) $Op1, $Op2, $Op3)>;
- def : Pat<(nxv4i32 (add nxv4i32:$Op2, (nxv4i32 (op nxv4i1:$Op1, (nxv4i32 (SVEDup0)), nxv8i16:$Op3)))),
- (!cast<Instruction>(NAME # _S) $Op1, $Op2, $Op3)>;
- def : Pat<(nxv2i64 (add nxv2i64:$Op2, (nxv2i64 (op nxv2i1:$Op1, (nxv2i64 (SVEDup0)), nxv4i32:$Op3)))),
- (!cast<Instruction>(NAME # _D) $Op1, $Op2, $Op3)>;
}
class sve2_int_un_pred_arit<bits<2> sz, bits<2> opc,
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-pairwise-arith.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-pairwise-arith.ll
index 9a1388a9bee04..7e852f7f1c287 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-pairwise-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-widening-pairwise-arith.ll
@@ -40,55 +40,6 @@ define <vscale x 2 x i64> @sadalp_i32(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
ret <vscale x 2 x i64> %out
}
-define <vscale x 8 x i16> @sadalp_i8_separate_add(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 16 x i8> %b) {
-; CHECK-LABEL: sadalp_i8_separate_add:
-; CHECK: // %bb.0:
-; CHECK-NEXT: sadalp z0.h, p0/m, z1.b
-; CHECK-NEXT: ret
- %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sadalp.nxv8i16(<vscale x 8 x i1> %pg,
- <vscale x 8 x i16> zeroinitializer,
- <vscale x 16 x i8> %b)
- %out2 = add <vscale x 8 x i16> %out, %a
- ret <vscale x 8 x i16> %out2
-}
-
-define <vscale x 4 x i32> @sadalp_i16_separate_add(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 8 x i16> %b) {
-; CHECK-LABEL: sadalp_i16_separate_add:
-; CHECK: // %bb.0:
-; CHECK-NEXT: sadalp z0.s, p0/m, z1.h
-; CHECK-NEXT: ret
- %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sadalp.nxv4i32(<vscale x 4 x i1> %pg,
- <vscale x 4 x i32> zeroinitializer,
- <vscale x 8 x i16> %b)
- %out2 = add <vscale x 4 x i32> %out, %a
- ret <vscale x 4 x i32> %out2
-}
-
-define <vscale x 2 x i64> @sadalp_i32_separate_add(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 4 x i32> %b) {
-; CHECK-LABEL: sadalp_i32_separate_add:
-; CHECK: // %bb.0:
-; CHECK-NEXT: sadalp z0.d, p0/m, z1.s
-; CHECK-NEXT: ret
- %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sadalp.nxv2i64(<vscale x 2 x i1> %pg,
- <vscale x 2 x i64> zeroinitializer,
- <vscale x 4 x i32> %b)
- %out2 = add <vscale x 2 x i64> %out, %a
- ret <vscale x 2 x i64> %out2
-}
-
-; Make sure the pattern is commutative
-define <vscale x 2 x i64> @sadalp_i32_separate_add_rev(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 4 x i32> %b) {
-; CHECK-LABEL: sadalp_i32_separate_add_rev:
-; CHECK: // %bb.0:
-; CHECK-NEXT: sadalp z0.d, p0/m, z1.s
-; CHECK-NEXT: ret
- %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sadalp.nxv2i64(<vscale x 2 x i1> %pg,
- <vscale x 2 x i64> zeroinitializer,
- <vscale x 4 x i32> %b)
- %out2 = add <vscale x 2 x i64> %a, %out
- ret <vscale x 2 x i64> %out2
-}
-
;
; UADALP
;
@@ -126,42 +77,6 @@ define <vscale x 2 x i64> @uadalp_i32(<vscale x 2 x i1> %pg, <vscale x 2 x i64>
ret <vscale x 2 x i64> %out
}
-define <vscale x 8 x i16> @uadalp_i8_separate_add(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 16 x i8> %b) {
-; CHECK-LABEL: uadalp_i8_separate_add:
-; CHECK: // %bb.0:
-; CHECK-NEXT: uadalp z0.h, p0/m, z1.b
-; CHECK-NEXT: ret
- %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uadalp.nxv8i16(<vscale x 8 x i1> %pg,
- <vscale x 8 x i16> zeroinitializer,
- <vscale x 16 x i8> %b)
- %out2 = add <vscale x 8 x i16> %out, %a
- ret <vscale x 8 x i16> %out2
-}
-
-define <vscale x 4 x i32> @uadalp_i16_separate_add(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 8 x i16> %b) {
-; CHECK-LABEL: uadalp_i16_separate_add:
-; CHECK: // %bb.0:
-; CHECK-NEXT: uadalp z0.s, p0/m, z1.h
-; CHECK-NEXT: ret
- %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uadalp.nxv4i32(<vscale x 4 x i1> %pg,
- <vscale x 4 x i32> zeroinitializer,
- <vscale x 8 x i16> %b)
- %out2 = add <vscale x 4 x i32> %out, %a
- ret <vscale x 4 x i32> %out2
-}
-
-define <vscale x 2 x i64> @uadalp_i32_separate_add(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 4 x i32> %b) {
-; CHECK-LABEL: uadalp_i32_separate_add:
-; CHECK: // %bb.0:
-; CHECK-NEXT: uadalp z0.d, p0/m, z1.s
-; CHECK-NEXT: ret
- %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uadalp.nxv2i64(<vscale x 2 x i1> %pg,
- <vscale x 2 x i64> zeroinitializer,
- <vscale x 4 x i32> %b)
- %out2 = add <vscale x 2 x i64> %out, %a
- ret <vscale x 2 x i64> %out2
-}
-
declare <vscale x 8 x i16> @llvm.aarch64.sve.sadalp.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 16 x i8>)
declare <vscale x 4 x i32> @llvm.aarch64.sve.sadalp.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 8 x i16>)
declare <vscale x 2 x i64> @llvm.aarch64.sve.sadalp.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 4 x i32>)
diff --git a/llvm/test/Transforms/InstCombine/AArch64/sve-pairwise-add-long.ll b/llvm/test/Transforms/InstCombine/AArch64/sve-pairwise-add-long.ll
new file mode 100644
index 0000000000000..c41ddce4b4d0e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AArch64/sve-pairwise-add-long.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=instcombine -mtriple=aarch64 < %s | FileCheck %s
+
+; When sadalp or uadalp accumulate into a zeroregister and they are followed by
+; an add instruction, the latter can be folded in.
+
+define <vscale x 8 x i16> @sadalp_i8_separate_add(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: define <vscale x 8 x i16> @sadalp_i8_separate_add(
+; CHECK-SAME: <vscale x 8 x i1> [[PG:%.*]], <vscale x 8 x i16> [[A:%.*]], <vscale x 16 x i8> [[B:%.*]]) {
+; CHECK-NEXT: [[OUT2:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.sadalp.nxv8i16(<vscale x 8 x i1> [[PG]], <vscale x 8 x i16> [[A]], <vscale x 16 x i8> [[B]])
+; CHECK-NEXT: ret <vscale x 8 x i16> [[OUT2]]
+;
+ %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sadalp.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> zeroinitializer, <vscale x 16 x i8> %b)
+ %out2 = add <vscale x 8 x i16> %out, %a
+ ret <vscale x 8 x i16> %out2
+}
+
+define <vscale x 4 x i32> @sadalp_i16_separate_add(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: define <vscale x 4 x i32> @sadalp_i16_separate_add(
+; CHECK-SAME: <vscale x 4 x i1> [[PG:%.*]], <vscale x 4 x i32> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]]) {
+; CHECK-NEXT: [[OUT2:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.sadalp.nxv4i32(<vscale x 4 x i1> [[PG]], <vscale x 4 x i32> [[A]], <vscale x 8 x i16> [[B]])
+; CHECK-NEXT: ret <vscale x 4 x i32> [[OUT2]]
+;
+ %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sadalp.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> zeroinitializer, <vscale x 8 x i16> %b)
+ %out2 = add <vscale x 4 x i32> %out, %a
+ ret <vscale x 4 x i32> %out2
+}
+
+define <vscale x 2 x i64> @sadalp_i32_separate_add(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: define <vscale x 2 x i64> @sadalp_i32_separate_add(
+; CHECK-SAME: <vscale x 2 x i1> [[PG:%.*]], <vscale x 2 x i64> [[A:%.*]], <vscale x 4 x i32> [[B:%.*]]) {
+; CHECK-NEXT: [[OUT2:%.*]] = call <vscale x 2 x i64> @llvm.aarch64.sve.sadalp.nxv2i64(<vscale x 2 x i1> [[PG]], <vscale x 2 x i64> [[A]], <vscale x 4 x i32> [[B]])
+; CHECK-NEXT: ret <vscale x 2 x i64> [[OUT2]]
+;
+ %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sadalp.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> zeroinitializer, <vscale x 4 x i32> %b)
+ %out2 = add <vscale x 2 x i64> %out, %a
+ ret <vscale x 2 x i64> %out2
+}
+
+define <vscale x 2 x i64> @sadalp_i32_separate_add_rev(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: define <vscale x 2 x i64> @sadalp_i32_separate_add_rev(
+; CHECK-SAME: <vscale x 2 x i1> [[PG:%.*]], <vscale x 2 x i64> [[A:%.*]], <vscale x 4 x i32> [[B:%.*]]) {
+; CHECK-NEXT: [[OUT2:%.*]] = call <vscale x 2 x i64> @llvm.aarch64.sve.sadalp.nxv2i64(<vscale x 2 x i1> [[PG]], <vscale x 2 x i64> [[A]], <vscale x 4 x i32> [[B]])
+; CHECK-NEXT: ret <vscale x 2 x i64> [[OUT2]]
+;
+ %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sadalp.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> zeroinitializer, <vscale x 4 x i32> %b)
+ %out2 = add <vscale x 2 x i64> %a, %out
+ ret <vscale x 2 x i64> %out2
+}
+
+define <vscale x 8 x i16> @uadalp_i8_separate_add(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: define <vscale x 8 x i16> @uadalp_i8_separate_add(
+; CHECK-SAME: <vscale x 8 x i1> [[PG:%.*]], <vscale x 8 x i16> [[A:%.*]], <vscale x 16 x i8> [[B:%.*]]) {
+; CHECK-NEXT: [[OUT2:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.uadalp.nxv8i16(<vscale x 8 x i1> [[PG]], <vscale x 8 x i16> [[A]], <vscale x 16 x i8> [[B]])
+; CHECK-NEXT: ret <vscale x 8 x i16> [[OUT2]]
+;
+ %out = call <vscale x 8 x i16> @llvm.aarch64.sve.uadalp.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> zeroinitializer, <vscale x 16 x i8> %b)
+ %out2 = add <vscale x 8 x i16> %out, %a
+ ret <vscale x 8 x i16> %out2
+}
+
+define <vscale x 4 x i32> @uadalp_i16_separate_add(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: define <vscale x 4 x i32> @uadalp_i16_separate_add(
+; CHECK-SAME: <vscale x 4 x i1> [[PG:%.*]], <vscale x 4 x i32> [[A:%.*]], <vscale x 8 x i16> [[B:%.*]]) {
+; CHECK-NEXT: [[OUT2:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.uadalp.nxv4i32(<vscale x 4 x i1> [[PG]], <vscale x 4 x i32> [[A]], <vscale x 8 x i16> [[B]])
+; CHECK-NEXT: ret <vscale x 4 x i32> [[OUT2]]
+;
+ %out = call <vscale x 4 x i32> @llvm.aarch64.sve.uadalp.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> zeroinitializer, <vscale x 8 x i16> %b)
+ %out2 = add <vscale x 4 x i32> %out, %a
+ ret <vscale x 4 x i32> %out2
+}
+
+define <vscale x 2 x i64> @uadalp_i32_separate_add(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: define <vscale x 2 x i64> @uadalp_i32_separate_add(
+; CHECK-SAME: <vscale x 2 x i1> [[PG:%.*]], <vscale x 2 x i64> [[A:%.*]], <vscale x 4 x i32> [[B:%.*]]) {
+; CHECK-NEXT: [[OUT2:%.*]] = call <vscale x 2 x i64> @llvm.aarch64.sve.uadalp.nxv2i64(<vscale x 2 x i1> [[PG]], <vscale x 2 x i64> [[A]], <vscale x 4 x i32> [[B]])
+; CHECK-NEXT: ret <vscale x 2 x i64> [[OUT2]]
+;
+ %out = call <vscale x 2 x i64> @llvm.aarch64.sve.uadalp.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> zeroinitializer, <vscale x 4 x i32> %b)
+ %out2 = add <vscale x 2 x i64> %out, %a
+ ret <vscale x 2 x i64> %out2
+}
+
+define <vscale x 2 x i64> @sadalp_nonzero_accumulator(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 4 x i32> %b, <vscale x 2 x i64> %c) {
+; CHECK-LABEL: define <vscale x 2 x i64> @sadalp_nonzero_accumulator(
+; CHECK-SAME: <vscale x 2 x i1> [[PG:%.*]], <vscale x 2 x i64> [[A:%.*]], <vscale x 4 x i32> [[B:%.*]], <vscale x 2 x i64> [[C:%.*]]) {
+; CHECK-NEXT: [[OUT:%.*]] = call <vscale x 2 x i64> @llvm.aarch64.sve.sadalp.nxv2i64(<vscale x 2 x i1> [[PG]], <vscale x 2 x i64> [[C]], <vscale x 4 x i32> [[B]])
+; CHECK-NEXT: [[OUT2:%.*]] = add <vscale x 2 x i64> [[OUT]], [[A]]
+; CHECK-NEXT: ret <vscale x 2 x i64> [[OUT2]]
+;
+ %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sadalp.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %c, <vscale x 4 x i32> %b)
+ %out2 = add <vscale x 2 x i64> %out, %a
+ ret <vscale x 2 x i64> %out2
+}
>From c0171a0598062d8941cfad36c6b61d00eb953a7c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ga=C3=ABtan=20Bossu?= <gaetan.bossu at arm.com>
Date: Thu, 2 Jul 2026 09:08:07 +0000
Subject: [PATCH 3/3] Comments
- use target triple in test
- remove both instructions early
- avoid dyn_cast and nullptr check
---
llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp | 9 +++++----
.../InstCombine/AArch64/sve-pairwise-add-long.ll | 4 +++-
2 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index b8799dd346874..2ffbc980ad9af 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -2608,19 +2608,20 @@ instCombineSVEPairwiseAddLong(InstCombiner &IC, IntrinsicInst &II) {
if (!II.hasOneUse() || !match(II.getArgOperand(1), m_Zero()))
return std::nullopt;
- auto *User = dyn_cast<Instruction>(*II.user_begin());
+ auto *User = cast<Instruction>(*II.user_begin());
Value *Acc;
- if (!User || !match(User, m_c_Add(m_Specific(&II), m_Value(Acc))))
+ if (!match(User, m_c_Add(m_Specific(&II), m_Value(Acc))))
return std::nullopt;
IC.Builder.SetInsertPoint(User);
Value *PairwiseAddLong = IC.Builder.CreateIntrinsic(
II.getIntrinsicID(), {II.getType()},
{II.getArgOperand(0), Acc, II.getArgOperand(2)});
- PairwiseAddLong->takeName(User);
IC.replaceInstUsesWith(*User, PairwiseAddLong);
- return IC.eraseInstFromFunction(*User);
+ IC.eraseInstFromFunction(*User);
+ IC.eraseInstFromFunction(II);
+ return nullptr;
}
static std::optional<Instruction *> instCombineSVEVectorAdd(InstCombiner &IC,
diff --git a/llvm/test/Transforms/InstCombine/AArch64/sve-pairwise-add-long.ll b/llvm/test/Transforms/InstCombine/AArch64/sve-pairwise-add-long.ll
index c41ddce4b4d0e..323ce50737aa3 100644
--- a/llvm/test/Transforms/InstCombine/AArch64/sve-pairwise-add-long.ll
+++ b/llvm/test/Transforms/InstCombine/AArch64/sve-pairwise-add-long.ll
@@ -1,5 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -S -passes=instcombine -mtriple=aarch64 < %s | FileCheck %s
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
; When sadalp or uadalp accumulate into a zeroregister and they are followed by
; an add instruction, the latter can be folded in.
More information about the llvm-commits
mailing list