[llvm] [AArch64] Use `AArch64ISD::UADDLP` for manual widening adjacent arithmetic (zext/shuffle combination) (PR #189255)
Rajveer Singh Bharadwaj via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 14 09:24:23 PDT 2026
https://github.com/Rajveer100 updated https://github.com/llvm/llvm-project/pull/189255
>From 785921e1274416fe0a00397988b1552bb08ebc06 Mon Sep 17 00:00:00 2001
From: Rajveer <rajveer.developer at icloud.com>
Date: Sun, 29 Mar 2026 21:20:24 +0530
Subject: [PATCH] [AArch64] Use `AArch64ISD::UADDLP` for manual widening
adjacent arithmetic (zext/shuffle combination)
Resolves #181490
This optimises the output for the following
patterns to `uaddlp(..., ...)`:
- add(shuffle_vec(zext(v0, ...), ...), shuffle_vec(zext(v0, ...), ...))
- add(zext(shuffle_vec(v0, ...), ...), zext(shuffle_vec(v0, ...), ...))
---
.../Target/AArch64/AArch64ISelLowering.cpp | 60 ++++++++++++++
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 4 +
llvm/test/CodeGen/AArch64/addp-shuffle.ll | 8 +-
llvm/test/CodeGen/AArch64/uaddlp.ll | 80 +++++++++++++++++++
4 files changed, 146 insertions(+), 6 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/uaddlp.ll
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 38db1ac4a2fb9..6d839350eaa1f 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -22310,6 +22310,64 @@ static SDValue performSetccAddFolding(SDNode *Op, SelectionDAG &DAG) {
return DAG.getNode(AArch64ISD::CSEL, DL, VT, RHS, LHS, CCVal, Cmp);
}
+static bool isEvenOddShufflePair(SDValue Op0, SDValue Op1) {
+ auto *S0 = dyn_cast<ShuffleVectorSDNode>(Op0);
+ auto *S1 = dyn_cast<ShuffleVectorSDNode>(Op1);
+
+ if (!S0 || !S1)
+ return false;
+
+ if (S0->getOperand(0) != S1->getOperand(0))
+ return false;
+
+ ArrayRef<int> Mask0 = S0->getMask();
+ ArrayRef<int> Mask1 = S1->getMask();
+
+ if (Mask0.size() != Mask1.size())
+ return false;
+
+ auto IsValidSequence = [](ArrayRef<int> Mask0, ArrayRef<int> Mask1) {
+ bool IsValid = true;
+ for (unsigned I = 0; I < Mask0.size(); ++I) {
+ if (Mask0[I] != (int)(I * 2) || Mask1[I] != (int)(I * 2 + 1))
+ IsValid = false;
+ }
+ return IsValid;
+ };
+
+ return IsValidSequence(Mask0, Mask1) || IsValidSequence(Mask1, Mask0);
+}
+
+static SDValue performVectorReduceWithExtAndShuffleToUADDLP(SDNode *N,
+ SelectionDAG &DAG) {
+ EVT VT = N->getValueType(0);
+ SDLoc DL(N);
+
+ if (N->getOpcode() != ISD::ADD)
+ return SDValue();
+
+ SDValue Op0 = N->getOperand(0);
+ SDValue Op1 = N->getOperand(1);
+
+ if (Op0.getOpcode() == ISD::VECTOR_SHUFFLE &&
+ Op1.getOpcode() == ISD::VECTOR_SHUFFLE) {
+ SDValue Z0 = Op0.getOperand(0);
+ SDValue Z1 = Op1.getOperand(0);
+
+ if (Z0 == Z1 && Z0.getOpcode() == ISD::ZERO_EXTEND) {
+ if (isEvenOddShufflePair(Op0, Op1)) {
+ SDValue Src = Z0.getOperand(0);
+ if (Src.getOpcode() == ISD::EXTRACT_SUBVECTOR)
+ Src = peekThroughExtractSubvectors(Z0.getOperand(0));
+
+ return DAG.getNode(AArch64ISD::UADDLP, DL, VT, Src);
+ }
+ }
+ }
+
+ return SDValue();
+}
+
// ADD(UADDV a, UADDV b) --> UADDV(ADD a, b)
static SDValue performAddUADDVCombine(SDNode *N, SelectionDAG &DAG) {
EVT VT = N->getValueType(0);
@@ -23416,6 +23474,8 @@ static SDValue performAddWithSBCCombine(SDNode *N, SelectionDAG &DAG) {
static SDValue performAddSubCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI) {
+ if (SDValue Val = performVectorReduceWithExtAndShuffleToUADDLP(N, DCI.DAG))
+ return Val;
// Try to change sum of two reductions.
if (SDValue Val = performAddUADDVCombine(N, DCI.DAG))
return Val;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 08512f6ed8df1..0b8788767ff2c 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -8641,6 +8641,10 @@ def : Pat<(v2i64 (AArch64saddlv (v2i32 V64:$Rn))),
def : Pat<(v2i64 (AArch64uaddlv (v2i32 V64:$Rn))),
(v2i64 (INSERT_SUBREG (v2i64 (IMPLICIT_DEF)), (UADDLPv2i32_v1i64 V64:$Rn), dsub))>;
+def : Pat<(add (AArch64bici (v8i16 (bitconvert v16i8:$src)), (i32 255), (i32 8)),
+ (AArch64vlshr (v8i16 (AArch64NvCast v16i8:$src)), (i32 8))),
+ (UADDLPv16i8_v8i16 V128:$src)>;
+
//------------------------------------------------------------------------------
// AdvSIMD modified immediate instructions
//------------------------------------------------------------------------------
diff --git a/llvm/test/CodeGen/AArch64/addp-shuffle.ll b/llvm/test/CodeGen/AArch64/addp-shuffle.ll
index 7ba01adad011c..0756ebffca5c4 100644
--- a/llvm/test/CodeGen/AArch64/addp-shuffle.ll
+++ b/llvm/test/CodeGen/AArch64/addp-shuffle.ll
@@ -160,12 +160,8 @@ define <4 x i32> @udot(<4 x i32> %z, <16 x i8> %a, <16 x i8> %b) {
; CHECK: // %bb.0:
; CHECK-NEXT: umull v3.8h, v1.8b, v2.8b
; CHECK-NEXT: umull2 v1.8h, v1.16b, v2.16b
-; CHECK-NEXT: ushll2 v2.4s, v3.8h, #0
-; CHECK-NEXT: ushll v3.4s, v3.4h, #0
-; CHECK-NEXT: ushll2 v4.4s, v1.8h, #0
-; CHECK-NEXT: ushll v1.4s, v1.4h, #0
-; CHECK-NEXT: addp v2.4s, v3.4s, v2.4s
-; CHECK-NEXT: addp v1.4s, v1.4s, v4.4s
+; CHECK-NEXT: uaddlp v1.4s, v1.8h
+; CHECK-NEXT: uaddlp v2.4s, v3.8h
; CHECK-NEXT: addp v1.4s, v2.4s, v1.4s
; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/uaddlp.ll b/llvm/test/CodeGen/AArch64/uaddlp.ll
new file mode 100644
index 0000000000000..00c60d78c3fef
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/uaddlp.ll
@@ -0,0 +1,80 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple aarch64-none-linux-gnu < %s | FileCheck %s
+
+define <8 x i16> @vpaddlq_u8_v1(<16 x i8> %a) {
+; CHECK-LABEL: vpaddlq_u8_v1:
+; CHECK: // %bb.0: // %start
+; CHECK-NEXT: uaddlp v0.8h, v0.16b
+; CHECK-NEXT: ret
+start:
+ %_0 = tail call <8 x i16> @llvm.aarch64.neon.uaddlp.v8i16.v16i8(<16 x i8> %a)
+ ret <8 x i16> %_0
+}
+
+define <8 x i16> @vpaddlq_u8_v2_widen_shuffle_add(<16 x i8> %a) {
+; CHECK-LABEL: vpaddlq_u8_v2_widen_shuffle_add:
+; CHECK: // %bb.0: // %start
+; CHECK-NEXT: uaddlp v0.8h, v0.16b
+; CHECK-NEXT: ret
+start:
+ %0 = zext <16 x i8> %a to <16 x i16>
+ %1 = shufflevector <16 x i16> %0, <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+ %2 = shufflevector <16 x i16> %0, <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+ %3 = add nuw nsw <8 x i16> %1, %2
+ ret <8 x i16> %3
+}
+
+define range(i16 0, 511) <8 x i16> @vpaddlq_u8_v3_shuffle_widen_add(<16 x i8> %a) {
+; CHECK-LABEL: vpaddlq_u8_v3_shuffle_widen_add:
+; CHECK: // %bb.0: // %start
+; CHECK-NEXT: uaddlp v0.8h, v0.16b
+; CHECK-NEXT: ret
+start:
+ %0 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+ %1 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+ %2 = zext <8 x i8> %0 to <8 x i16>
+ %3 = zext <8 x i8> %1 to <8 x i16>
+ %4 = add nuw nsw <8 x i16> %2, %3
+ ret <8 x i16> %4
+}
+
+define <8 x i16> @vpaddlq_u8_v2_widen_shuffle_add_neg(<16 x i8> %a) {
+; CHECK-LABEL: vpaddlq_u8_v2_widen_shuffle_add_neg:
+; CHECK: // %bb.0: // %start
+; CHECK-NEXT: ushll2 v2.8h, v0.16b, #0
+; CHECK-NEXT: adrp x8, .LCPI3_0
+; CHECK-NEXT: adrp x9, .LCPI3_1
+; CHECK-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI3_0]
+; CHECK-NEXT: ldr q3, [x9, :lo12:.LCPI3_1]
+; CHECK-NEXT: tbl v0.16b, { v1.16b, v2.16b }, v0.16b
+; CHECK-NEXT: tbl v1.16b, { v1.16b, v2.16b }, v3.16b
+; CHECK-NEXT: add v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: ret
+start:
+ %0 = zext <16 x i8> %a to <16 x i16>
+ %1 = shufflevector <16 x i16> %0, <16 x i16> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 11, i32 10, i32 12, i32 14>
+ %2 = shufflevector <16 x i16> %0, <16 x i16> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 8, i32 13, i32 15>
+ %3 = add nuw nsw <8 x i16> %1, %2
+ ret <8 x i16> %3
+}
+
+define range(i16 0, 511) <8 x i16> @vpaddlq_u8_v3_shuffle_widen_add_neg(<16 x i8> %a) {
+; CHECK-LABEL: vpaddlq_u8_v3_shuffle_widen_add_neg:
+; CHECK: // %bb.0: // %start
+; CHECK-NEXT: adrp x8, .LCPI4_0
+; CHECK-NEXT: adrp x9, .LCPI4_1
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI4_0]
+; CHECK-NEXT: ldr d2, [x9, :lo12:.LCPI4_1]
+; CHECK-NEXT: tbl v1.8b, { v0.16b }, v1.8b
+; CHECK-NEXT: tbl v0.8b, { v0.16b }, v2.8b
+; CHECK-NEXT: uaddl v0.8h, v1.8b, v0.8b
+; CHECK-NEXT: ret
+start:
+ %0 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 0, i32 3, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+ %1 = shufflevector <16 x i8> %a, <16 x i8> poison, <8 x i32> <i32 1, i32 2, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+ %2 = zext <8 x i8> %0 to <8 x i16>
+ %3 = zext <8 x i8> %1 to <8 x i16>
+ %4 = add nuw nsw <8 x i16> %2, %3
+ ret <8 x i16> %4
+}
More information about the llvm-commits
mailing list