[llvm] [AArch64] Fold zero-interleave shuffle into vector shift left (PR #210793)
Deepak Shirke via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 31 06:02:48 PDT 2026
https://github.com/deepakshirkem updated https://github.com/llvm/llvm-project/pull/210793
>From 6702fca14f10f50f8db1328242dd3cc0e4f4197c Mon Sep 17 00:00:00 2001
From: deepakshirkem <deepakshirke509 at gmail.com>
Date: Wed, 29 Jul 2026 15:23:13 +0530
Subject: [PATCH 1/2] [AArch64] Generalise isSlideWithZerosMask to handle
smaller lane sizes
Extend isSlideWithZerosMask to try lane sizes of 64, 32 and 16 bits
instead of only 64 bits. This allows it to recognise shuffle patterns
that represent sub-element slides, such as zero-interleave patterns
that are equivalent to vector shifts on wider element types.
For example, <16 x i8> shuffle [N,0,N,2,N,4,...] (N>=NumElts) is now
recognised as a 16-bit lane slide by 1 element, equivalent to:
shl v0.8h, v0.8h, #8
Before:
adrp x8, .LCPI0_0
ldr q1, [x8, :lo12:.LCPI0_0]
tbl v0.16b, { v0.16b }, v1.16b
After:
shl v0.8h, v0.8h, #8
---
.../Target/AArch64/AArch64ISelLowering.cpp | 64 +++++++++++--------
.../AArch64/shuffle-zero-interleave-to-shl.ll | 36 +++++++++++
2 files changed, 75 insertions(+), 25 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/shuffle-zero-interleave-to-shl.ll
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index a5a2c9e430ac2..369529ae5f8fc 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -15202,31 +15202,39 @@ static SDValue isSlideWithZerosMask(ArrayRef<int> M, EVT VT, SDValue V1,
DataVec = V2;
}
- // For 64-bit vectors, check single lane
- // For 128-bit vectors, check both 64-bit lanes have same slide
- unsigned LaneElts = 64 / EltSize;
- unsigned NumLanes = VTSize / 64;
-
- bool FirstIsLeftSlide;
- unsigned FirstSlideAmt =
- checkLaneSlide(Mask, 0, LaneElts, NumElts, FirstIsLeftSlide);
- if (FirstSlideAmt == 0)
- return SDValue();
-
- // For 128-bit, verify second lane matches
- if (NumLanes == 2) {
- bool SecondIsLeftSlide;
- unsigned SecondSlideAmt =
- checkLaneSlide(Mask, LaneElts, LaneElts, NumElts, SecondIsLeftSlide);
- if (SecondSlideAmt != FirstSlideAmt ||
- SecondIsLeftSlide != FirstIsLeftSlide)
- return SDValue();
- }
+ // Try lane sizes 64, 32, 16 bits.
+ // For each lane size, check all lanes have the same slide pattern.
+ for (unsigned LaneSize : {64u, 32u, 16u}) {
+ if (LaneSize < EltSize * 2)
+ break; // need at least 2 elements per lane
+ unsigned LaneElts = LaneSize / EltSize;
+ unsigned NumLanes = VTSize / LaneSize;
+
+ bool FirstIsLeftSlide;
+ unsigned FirstSlideAmt =
+ checkLaneSlide(Mask, 0, LaneElts, NumElts, FirstIsLeftSlide);
+ if (FirstSlideAmt == 0)
+ continue;
+
+ // Verify all lanes match
+ bool AllMatch = true;
+ for (unsigned Lane = 1; Lane < NumLanes; Lane++) {
+ bool IsLeftSlide;
+ unsigned SlideAmt =
+ checkLaneSlide(Mask, Lane * LaneElts, LaneElts, NumElts, IsLeftSlide);
+ if (SlideAmt != FirstSlideAmt || IsLeftSlide != FirstIsLeftSlide) {
+ AllMatch = false;
+ break;
+ }
+ }
+ if (!AllMatch)
+ continue;
- ShiftAmount = FirstSlideAmt * EltSize;
- IsRightShift = FirstIsLeftSlide; // left slide = right shift in bits
- if (ShiftAmount > 0 && ShiftAmount < 64)
- return DataVec;
+ ShiftAmount = FirstSlideAmt * EltSize;
+ IsRightShift = FirstIsLeftSlide;
+ if (ShiftAmount > 0 && ShiftAmount < LaneSize)
+ return DataVec;
+ }
return SDValue();
}
@@ -15991,7 +15999,13 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
bool IsRightShift;
if (SDValue DataVec = isSlideWithZerosMask(ShuffleMask, VT, V1, V2,
ShiftAmount, IsRightShift)) {
- MVT ShiftVT = VT.getSizeInBits() == 64 ? MVT::v1i64 : MVT::v2i64;
+ MVT ShiftVT;
+ if (ShiftAmount >= 32)
+ ShiftVT = VT.getSizeInBits() == 64 ? MVT::v1i64 : MVT::v2i64;
+ else if (ShiftAmount >= 16)
+ ShiftVT = VT.getSizeInBits() == 64 ? MVT::v2i32 : MVT::v4i32;
+ else
+ ShiftVT = VT.getSizeInBits() == 64 ? MVT::v4i16 : MVT::v8i16;
SDValue Vec = DAG.getNode(AArch64ISD::NVCAST, DL, ShiftVT, DataVec);
SDValue ShiftAmt = DAG.getTargetConstant(ShiftAmount, DL, MVT::i32);
diff --git a/llvm/test/CodeGen/AArch64/shuffle-zero-interleave-to-shl.ll b/llvm/test/CodeGen/AArch64/shuffle-zero-interleave-to-shl.ll
new file mode 100644
index 0000000000000..0bb581fb82ebd
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/shuffle-zero-interleave-to-shl.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc < %s -mtriple=aarch64 | FileCheck %s
+
+; Shuffle patterns that interleave zeros between bytes are equivalent to
+; vector shift left on wider element type. These should use shl
+; instead of tbl with a constant mask loaded from memory.
+
+; SHL pattern: zeros in even positions, data in odd positions
+define <16 x i8> @shl_v16i8_data_first(<16 x i8> %v) {
+; CHECK-LABEL: shl_v16i8_data_first:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.8h, v0.8h, #8
+; CHECK-NEXT: ret
+ %r = shufflevector <16 x i8> %v, <16 x i8> zeroinitializer, <16 x i32> <i32 16, i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14>
+ ret <16 x i8> %r
+}
+
+; SHL pattern: zeros in first vector, data in second vector
+define <16 x i8> @shl_v16i8_zeros_first(<16 x i8> %v) {
+; CHECK-LABEL: shl_v16i8_zeros_first:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.8h, v0.8h, #8
+; CHECK-NEXT: ret
+ %r = shufflevector <16 x i8> zeroinitializer, <16 x i8> %v, <16 x i32> <i32 0, i32 16, i32 0, i32 18, i32 0, i32 20, i32 0, i32 22, i32 0, i32 24, i32 0, i32 26, i32 0, i32 28, i32 0, i32 30>
+ ret <16 x i8> %r
+}
+
+; v8i8 case
+define <8 x i8> @shl_v8i8(<8 x i8> %v) {
+; CHECK-LABEL: shl_v8i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-NEXT: ret
+ %r = shufflevector <8 x i8> %v, <8 x i8> zeroinitializer, <8 x i32> <i32 8, i32 0, i32 8, i32 2, i32 8, i32 4, i32 8, i32 6>
+ ret <8 x i8> %r
+}
>From 6a8e6dd2f25a16bf86e4e5e8d32a3dbb2b70c045 Mon Sep 17 00:00:00 2001
From: deepakshirkem <deepakshirke509 at gmail.com>
Date: Fri, 31 Jul 2026 18:30:00 +0530
Subject: [PATCH 2/2] [AArch64] Pass MatchedLaneSize from isSlideWithZerosMask
and add more tests
Pass the matched lane size directly from isSlideWithZerosMask to the
call site instead of deriving the shift type from ShiftAmount, which
could be ambiguous (e.g. a shift of 8 could apply to i16 or i32 lanes).
Also add test cases for i16 and i32 element vectors.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 22 +++++++++----------
.../AArch64/shuffle-zero-interleave-to-shl.ll | 20 +++++++++++++++++
2 files changed, 31 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 369529ae5f8fc..cc1f3d799e7e7 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -15179,7 +15179,8 @@ static unsigned checkLaneSlide(ArrayRef<int> Mask, unsigned LaneStart,
static SDValue isSlideWithZerosMask(ArrayRef<int> M, EVT VT, SDValue V1,
SDValue V2, unsigned &ShiftAmount,
- bool &IsRightShift) {
+ bool &IsRightShift,
+ unsigned &MatchedLaneSize) {
unsigned VTSize = VT.getSizeInBits();
if (VTSize != 64 && VTSize != 128)
return SDValue();
@@ -15232,8 +15233,10 @@ static SDValue isSlideWithZerosMask(ArrayRef<int> M, EVT VT, SDValue V1,
ShiftAmount = FirstSlideAmt * EltSize;
IsRightShift = FirstIsLeftSlide;
- if (ShiftAmount > 0 && ShiftAmount < LaneSize)
+ if (ShiftAmount > 0 && ShiftAmount < LaneSize) {
+ MatchedLaneSize = LaneSize;
return DataVec;
+ }
}
return SDValue();
}
@@ -15997,15 +16000,12 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
{
unsigned ShiftAmount;
bool IsRightShift;
- if (SDValue DataVec = isSlideWithZerosMask(ShuffleMask, VT, V1, V2,
- ShiftAmount, IsRightShift)) {
- MVT ShiftVT;
- if (ShiftAmount >= 32)
- ShiftVT = VT.getSizeInBits() == 64 ? MVT::v1i64 : MVT::v2i64;
- else if (ShiftAmount >= 16)
- ShiftVT = VT.getSizeInBits() == 64 ? MVT::v2i32 : MVT::v4i32;
- else
- ShiftVT = VT.getSizeInBits() == 64 ? MVT::v4i16 : MVT::v8i16;
+ unsigned MatchedLaneSize;
+ if (SDValue DataVec =
+ isSlideWithZerosMask(ShuffleMask, VT, V1, V2, ShiftAmount,
+ IsRightShift, MatchedLaneSize)) {
+ MVT ShiftVT = MVT::getVectorVT(MVT::getIntegerVT(MatchedLaneSize),
+ VT.getSizeInBits() / MatchedLaneSize);
SDValue Vec = DAG.getNode(AArch64ISD::NVCAST, DL, ShiftVT, DataVec);
SDValue ShiftAmt = DAG.getTargetConstant(ShiftAmount, DL, MVT::i32);
diff --git a/llvm/test/CodeGen/AArch64/shuffle-zero-interleave-to-shl.ll b/llvm/test/CodeGen/AArch64/shuffle-zero-interleave-to-shl.ll
index 0bb581fb82ebd..db508d9fb256d 100644
--- a/llvm/test/CodeGen/AArch64/shuffle-zero-interleave-to-shl.ll
+++ b/llvm/test/CodeGen/AArch64/shuffle-zero-interleave-to-shl.ll
@@ -34,3 +34,23 @@ define <8 x i8> @shl_v8i8(<8 x i8> %v) {
%r = shufflevector <8 x i8> %v, <8 x i8> zeroinitializer, <8 x i32> <i32 8, i32 0, i32 8, i32 2, i32 8, i32 4, i32 8, i32 6>
ret <8 x i8> %r
}
+
+; i16 element case: <8 x i16> zero-interleave --> shl <4 x i32>, #16
+define <8 x i16> @shl_v8i16(<8 x i16> %v) {
+; CHECK-LABEL: shl_v8i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.4s, v0.4s, #16
+; CHECK-NEXT: ret
+ %r = shufflevector <8 x i16> %v, <8 x i16> zeroinitializer, <8 x i32> <i32 8, i32 0, i32 8, i32 2, i32 8, i32 4, i32 8, i32 6>
+ ret <8 x i16> %r
+}
+
+; i32 element case: <4 x i32> zero-interleave --> shl <2 x i64>, #32
+define <4 x i32> @shl_v4i32(<4 x i32> %v) {
+; CHECK-LABEL: shl_v4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.2d, v0.2d, #32
+; CHECK-NEXT: ret
+ %r = shufflevector <4 x i32> %v, <4 x i32> zeroinitializer, <4 x i32> <i32 4, i32 0, i32 4, i32 2>
+ ret <4 x i32> %r
+}
More information about the llvm-commits
mailing list