[llvm] [AArch64] Fold zero-interleave shuffle into vector shift left (PR #210793)
Deepak Shirke via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 29 02:49:23 PDT 2026
https://github.com/deepakshirkem updated https://github.com/llvm/llvm-project/pull/210793
>From 96e798e49e1e6a9d99aa2fee584d92869686b960 Mon Sep 17 00:00:00 2001
From: deepakshirkem <deepakshirke509 at gmail.com>
Date: Tue, 21 Jul 2026 00:53:44 +0530
Subject: [PATCH 1/3] [AArch64] Fold zero-interleave shuffle into vector shift
left
A shuffle mask that interleaves zeros between every other byte element
is equivalent to a vector shift left on a wider element type.
For example, <16 x i8> shuffle [N,0,N,2,N,4,...] (N>=NumElts, zero src)
is equivalent to shl <8 x i16>, #8.
This avoids generating a tbl instruction with a constant mask loaded
from memory, replacing it with a single shl instruction.
Before:
adrp x8, .LCPI0_0
ldr q1, [x8, :lo12:.LCPI0_0]
tbl v0.16b, { v0.16b }, v1.16b
After:
shl v0.8h, v0.8h, #8
---
.../Target/AArch64/AArch64ISelLowering.cpp | 72 +++++++++++++++++++
.../AArch64/shuffle-zero-interleave-to-shl.ll | 36 ++++++++++
2 files changed, 108 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/shuffle-zero-interleave-to-shl.ll
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index a5a2c9e430ac2..f3c8bfcc99699 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -15177,6 +15177,62 @@ static unsigned checkLaneSlide(ArrayRef<int> Mask, unsigned LaneStart,
return 0;
}
+/// Check if a shuffle mask represents zero-interleaving of every other element,
+/// which is equivalent to a vector shift on a wider element type.
+/// e.g. <16 x i8> shuffle [N,0,N,2,N,4,...] --> shl <8 x i16>, #8
+/// <16 x i8> shuffle [0,N,2,N,4,N,...] --> lshr <8 x i16>, #8
+static unsigned isZeroInterleaveMask(ArrayRef<int> Mask, EVT VT,
+ SDValue V1, SDValue V2,
+ SDValue &DataVec, bool &IsLeftShift) {
+ if (VT != MVT::v16i8 && VT != MVT::v8i8)
+ return 0;
+ unsigned NumElts = Mask.size();
+ if (NumElts % 2 != 0)
+ return 0;
+
+ bool V1IsZeros = ISD::isBuildVectorAllZeros(V1.getNode());
+ bool V2IsZeros = ISD::isBuildVectorAllZeros(V2.getNode());
+ if (V1IsZeros == V2IsZeros)
+ return 0;
+
+ // Canonicalize so that V2 is zeros and V1 is data
+ SmallVector<int, 16> CanonMask(Mask.begin(), Mask.end());
+ DataVec = V1;
+ if (V1IsZeros) {
+ ShuffleVectorSDNode::commuteMask(CanonMask);
+ DataVec = V2;
+ }
+
+ // After canonicalization V2 is zeros (indices >= NumElts)
+ // Check SHL pattern: [N, 0, N, 2, N, 4, ...] where N >= NumElts
+ bool IsSHL = true;
+ for (unsigned i = 0; i < NumElts; i++) {
+ if (i % 2 == 0) {
+ if (CanonMask[i] >= 0 && CanonMask[i] < (int)NumElts)
+ { IsSHL = false; break; }
+ } else {
+ if (CanonMask[i] != (int)(i - 1))
+ { IsSHL = false; break; }
+ }
+ }
+ if (IsSHL) { IsLeftShift = true; return 8; }
+
+ // Check SHR pattern: [0, N, 2, N, 4, N, ...] where N >= NumElts
+ bool IsSHR = true;
+ for (unsigned i = 0; i < NumElts; i++) {
+ if (i % 2 == 1) {
+ if (CanonMask[i] >= 0 && CanonMask[i] < (int)NumElts)
+ { IsSHR = false; break; }
+ } else {
+ if (CanonMask[i] != (int)i)
+ { IsSHR = false; break; }
+ }
+ }
+ if (IsSHR) { IsLeftShift = false; return 8; }
+
+ return 0;
+}
+
static SDValue isSlideWithZerosMask(ArrayRef<int> M, EVT VT, SDValue V1,
SDValue V2, unsigned &ShiftAmount,
bool &IsRightShift) {
@@ -15985,6 +16041,22 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
DAG.getConstant(8, DL, MVT::i32));
}
+ // Check for zero-interleave pattern equivalent to vector shift on wider type.
+ // e.g. <16 x i8> shuffle [N,0,N,2,N,4,...] --> shl <8 x i16>, #8
+ {
+ bool IsLeftShift;
+ SDValue DataVec;
+ if (unsigned ShiftAmt = isZeroInterleaveMask(ShuffleMask, VT, V1, V2,
+ DataVec, IsLeftShift)) {
+ MVT WideVT = VT == MVT::v16i8 ? MVT::v8i16 : MVT::v4i16;
+ SDValue Vec = DAG.getBitcast(WideVT, DataVec);
+ SDValue Result = DAG.getNode(IsLeftShift ? ISD::SHL : ISD::SRL,
+ DL, WideVT, Vec,
+ DAG.getConstant(ShiftAmt, DL, WideVT));
+ return DAG.getBitcast(VT, Result);
+ }
+ }
+
// Check for slide-with-zeros pattern before EXT (slide is also valid EXT)
{
unsigned ShiftAmount;
diff --git a/llvm/test/CodeGen/AArch64/shuffle-zero-interleave-to-shl.ll b/llvm/test/CodeGen/AArch64/shuffle-zero-interleave-to-shl.ll
new file mode 100644
index 0000000000000..0bb581fb82ebd
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/shuffle-zero-interleave-to-shl.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc < %s -mtriple=aarch64 | FileCheck %s
+
+; Shuffle patterns that interleave zeros between bytes are equivalent to
+; vector shift left on wider element type. These should use shl
+; instead of tbl with a constant mask loaded from memory.
+
+; SHL pattern: zeros in even positions, data in odd positions
+define <16 x i8> @shl_v16i8_data_first(<16 x i8> %v) {
+; CHECK-LABEL: shl_v16i8_data_first:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.8h, v0.8h, #8
+; CHECK-NEXT: ret
+ %r = shufflevector <16 x i8> %v, <16 x i8> zeroinitializer, <16 x i32> <i32 16, i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14>
+ ret <16 x i8> %r
+}
+
+; SHL pattern: zeros in first vector, data in second vector
+define <16 x i8> @shl_v16i8_zeros_first(<16 x i8> %v) {
+; CHECK-LABEL: shl_v16i8_zeros_first:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.8h, v0.8h, #8
+; CHECK-NEXT: ret
+ %r = shufflevector <16 x i8> zeroinitializer, <16 x i8> %v, <16 x i32> <i32 0, i32 16, i32 0, i32 18, i32 0, i32 20, i32 0, i32 22, i32 0, i32 24, i32 0, i32 26, i32 0, i32 28, i32 0, i32 30>
+ ret <16 x i8> %r
+}
+
+; v8i8 case
+define <8 x i8> @shl_v8i8(<8 x i8> %v) {
+; CHECK-LABEL: shl_v8i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-NEXT: ret
+ %r = shufflevector <8 x i8> %v, <8 x i8> zeroinitializer, <8 x i32> <i32 8, i32 0, i32 8, i32 2, i32 8, i32 4, i32 8, i32 6>
+ ret <8 x i8> %r
+}
>From 6646496e544a72eccf0ff51280f3bf6b79e39b0d Mon Sep 17 00:00:00 2001
From: deepakshirkem <deepakshirke509 at gmail.com>
Date: Tue, 21 Jul 2026 01:00:28 +0530
Subject: [PATCH 2/3] [AArch64] clang-format fixes
---
.../Target/AArch64/AArch64ISelLowering.cpp | 46 ++++++++++++-------
1 file changed, 30 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index f3c8bfcc99699..ebd12048ce0fd 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -15181,9 +15181,9 @@ static unsigned checkLaneSlide(ArrayRef<int> Mask, unsigned LaneStart,
/// which is equivalent to a vector shift on a wider element type.
/// e.g. <16 x i8> shuffle [N,0,N,2,N,4,...] --> shl <8 x i16>, #8
/// <16 x i8> shuffle [0,N,2,N,4,N,...] --> lshr <8 x i16>, #8
-static unsigned isZeroInterleaveMask(ArrayRef<int> Mask, EVT VT,
- SDValue V1, SDValue V2,
- SDValue &DataVec, bool &IsLeftShift) {
+static unsigned isZeroInterleaveMask(ArrayRef<int> Mask, EVT VT, SDValue V1,
+ SDValue V2, SDValue &DataVec,
+ bool &IsLeftShift) {
if (VT != MVT::v16i8 && VT != MVT::v8i8)
return 0;
unsigned NumElts = Mask.size();
@@ -15208,27 +15208,41 @@ static unsigned isZeroInterleaveMask(ArrayRef<int> Mask, EVT VT,
bool IsSHL = true;
for (unsigned i = 0; i < NumElts; i++) {
if (i % 2 == 0) {
- if (CanonMask[i] >= 0 && CanonMask[i] < (int)NumElts)
- { IsSHL = false; break; }
+ if (CanonMask[i] >= 0 && CanonMask[i] < (int)NumElts) {
+ IsSHL = false;
+ break;
+ }
} else {
- if (CanonMask[i] != (int)(i - 1))
- { IsSHL = false; break; }
+ if (CanonMask[i] != (int)(i - 1)) {
+ IsSHL = false;
+ break;
+ }
}
}
- if (IsSHL) { IsLeftShift = true; return 8; }
+ if (IsSHL) {
+ IsLeftShift = true;
+ return 8;
+ }
// Check SHR pattern: [0, N, 2, N, 4, N, ...] where N >= NumElts
bool IsSHR = true;
for (unsigned i = 0; i < NumElts; i++) {
if (i % 2 == 1) {
- if (CanonMask[i] >= 0 && CanonMask[i] < (int)NumElts)
- { IsSHR = false; break; }
+ if (CanonMask[i] >= 0 && CanonMask[i] < (int)NumElts) {
+ IsSHR = false;
+ break;
+ }
} else {
- if (CanonMask[i] != (int)i)
- { IsSHR = false; break; }
+ if (CanonMask[i] != (int)i) {
+ IsSHR = false;
+ break;
+ }
}
}
- if (IsSHR) { IsLeftShift = false; return 8; }
+ if (IsSHR) {
+ IsLeftShift = false;
+ return 8;
+ }
return 0;
}
@@ -16050,9 +16064,9 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
DataVec, IsLeftShift)) {
MVT WideVT = VT == MVT::v16i8 ? MVT::v8i16 : MVT::v4i16;
SDValue Vec = DAG.getBitcast(WideVT, DataVec);
- SDValue Result = DAG.getNode(IsLeftShift ? ISD::SHL : ISD::SRL,
- DL, WideVT, Vec,
- DAG.getConstant(ShiftAmt, DL, WideVT));
+ SDValue Result =
+ DAG.getNode(IsLeftShift ? ISD::SHL : ISD::SRL, DL, WideVT, Vec,
+ DAG.getConstant(ShiftAmt, DL, WideVT));
return DAG.getBitcast(VT, Result);
}
}
>From 1cbb7479a320d4465fa3e5431a85aa9150462fc9 Mon Sep 17 00:00:00 2001
From: deepakshirkem <deepakshirke509 at gmail.com>
Date: Wed, 29 Jul 2026 00:27:48 +0530
Subject: [PATCH 3/3] [AArch64] Generalise isSlideWithZerosMask to handle
smaller lane sizes
Extend isSlideWithZerosMask to try lane sizes of 64, 32 and 16 bits
instead of only 64 bits. This allows it to recognise shuffle patterns
that represent sub-element slides, such as zero-interleave patterns
that are equivalent to vector shifts on wider element types.
For example, <16 x i8> shuffle [N,0,N,2,N,4,...] (N>=NumElts) is now
recognised as a 16-bit lane slide by 1 element, equivalent to:
shl v0.8h, v0.8h, #8
Before:
adrp x8, .LCPI0_0
ldr q1, [x8, :lo12:.LCPI0_0]
tbl v0.16b, { v0.16b }, v1.16b
After:
shl v0.8h, v0.8h, #8
---
.../Target/AArch64/AArch64ISelLowering.cpp | 146 +++++-------------
1 file changed, 37 insertions(+), 109 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index ebd12048ce0fd..369529ae5f8fc 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -15177,76 +15177,6 @@ static unsigned checkLaneSlide(ArrayRef<int> Mask, unsigned LaneStart,
return 0;
}
-/// Check if a shuffle mask represents zero-interleaving of every other element,
-/// which is equivalent to a vector shift on a wider element type.
-/// e.g. <16 x i8> shuffle [N,0,N,2,N,4,...] --> shl <8 x i16>, #8
-/// <16 x i8> shuffle [0,N,2,N,4,N,...] --> lshr <8 x i16>, #8
-static unsigned isZeroInterleaveMask(ArrayRef<int> Mask, EVT VT, SDValue V1,
- SDValue V2, SDValue &DataVec,
- bool &IsLeftShift) {
- if (VT != MVT::v16i8 && VT != MVT::v8i8)
- return 0;
- unsigned NumElts = Mask.size();
- if (NumElts % 2 != 0)
- return 0;
-
- bool V1IsZeros = ISD::isBuildVectorAllZeros(V1.getNode());
- bool V2IsZeros = ISD::isBuildVectorAllZeros(V2.getNode());
- if (V1IsZeros == V2IsZeros)
- return 0;
-
- // Canonicalize so that V2 is zeros and V1 is data
- SmallVector<int, 16> CanonMask(Mask.begin(), Mask.end());
- DataVec = V1;
- if (V1IsZeros) {
- ShuffleVectorSDNode::commuteMask(CanonMask);
- DataVec = V2;
- }
-
- // After canonicalization V2 is zeros (indices >= NumElts)
- // Check SHL pattern: [N, 0, N, 2, N, 4, ...] where N >= NumElts
- bool IsSHL = true;
- for (unsigned i = 0; i < NumElts; i++) {
- if (i % 2 == 0) {
- if (CanonMask[i] >= 0 && CanonMask[i] < (int)NumElts) {
- IsSHL = false;
- break;
- }
- } else {
- if (CanonMask[i] != (int)(i - 1)) {
- IsSHL = false;
- break;
- }
- }
- }
- if (IsSHL) {
- IsLeftShift = true;
- return 8;
- }
-
- // Check SHR pattern: [0, N, 2, N, 4, N, ...] where N >= NumElts
- bool IsSHR = true;
- for (unsigned i = 0; i < NumElts; i++) {
- if (i % 2 == 1) {
- if (CanonMask[i] >= 0 && CanonMask[i] < (int)NumElts) {
- IsSHR = false;
- break;
- }
- } else {
- if (CanonMask[i] != (int)i) {
- IsSHR = false;
- break;
- }
- }
- }
- if (IsSHR) {
- IsLeftShift = false;
- return 8;
- }
-
- return 0;
-}
-
static SDValue isSlideWithZerosMask(ArrayRef<int> M, EVT VT, SDValue V1,
SDValue V2, unsigned &ShiftAmount,
bool &IsRightShift) {
@@ -15272,31 +15202,39 @@ static SDValue isSlideWithZerosMask(ArrayRef<int> M, EVT VT, SDValue V1,
DataVec = V2;
}
- // For 64-bit vectors, check single lane
- // For 128-bit vectors, check both 64-bit lanes have same slide
- unsigned LaneElts = 64 / EltSize;
- unsigned NumLanes = VTSize / 64;
+ // Try lane sizes 64, 32, 16 bits.
+ // For each lane size, check all lanes have the same slide pattern.
+ for (unsigned LaneSize : {64u, 32u, 16u}) {
+ if (LaneSize < EltSize * 2)
+ break; // need at least 2 elements per lane
+ unsigned LaneElts = LaneSize / EltSize;
+ unsigned NumLanes = VTSize / LaneSize;
- bool FirstIsLeftSlide;
- unsigned FirstSlideAmt =
- checkLaneSlide(Mask, 0, LaneElts, NumElts, FirstIsLeftSlide);
- if (FirstSlideAmt == 0)
- return SDValue();
+ bool FirstIsLeftSlide;
+ unsigned FirstSlideAmt =
+ checkLaneSlide(Mask, 0, LaneElts, NumElts, FirstIsLeftSlide);
+ if (FirstSlideAmt == 0)
+ continue;
- // For 128-bit, verify second lane matches
- if (NumLanes == 2) {
- bool SecondIsLeftSlide;
- unsigned SecondSlideAmt =
- checkLaneSlide(Mask, LaneElts, LaneElts, NumElts, SecondIsLeftSlide);
- if (SecondSlideAmt != FirstSlideAmt ||
- SecondIsLeftSlide != FirstIsLeftSlide)
- return SDValue();
- }
+ // Verify all lanes match
+ bool AllMatch = true;
+ for (unsigned Lane = 1; Lane < NumLanes; Lane++) {
+ bool IsLeftSlide;
+ unsigned SlideAmt =
+ checkLaneSlide(Mask, Lane * LaneElts, LaneElts, NumElts, IsLeftSlide);
+ if (SlideAmt != FirstSlideAmt || IsLeftSlide != FirstIsLeftSlide) {
+ AllMatch = false;
+ break;
+ }
+ }
+ if (!AllMatch)
+ continue;
- ShiftAmount = FirstSlideAmt * EltSize;
- IsRightShift = FirstIsLeftSlide; // left slide = right shift in bits
- if (ShiftAmount > 0 && ShiftAmount < 64)
- return DataVec;
+ ShiftAmount = FirstSlideAmt * EltSize;
+ IsRightShift = FirstIsLeftSlide;
+ if (ShiftAmount > 0 && ShiftAmount < LaneSize)
+ return DataVec;
+ }
return SDValue();
}
@@ -16055,29 +15993,19 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
DAG.getConstant(8, DL, MVT::i32));
}
- // Check for zero-interleave pattern equivalent to vector shift on wider type.
- // e.g. <16 x i8> shuffle [N,0,N,2,N,4,...] --> shl <8 x i16>, #8
- {
- bool IsLeftShift;
- SDValue DataVec;
- if (unsigned ShiftAmt = isZeroInterleaveMask(ShuffleMask, VT, V1, V2,
- DataVec, IsLeftShift)) {
- MVT WideVT = VT == MVT::v16i8 ? MVT::v8i16 : MVT::v4i16;
- SDValue Vec = DAG.getBitcast(WideVT, DataVec);
- SDValue Result =
- DAG.getNode(IsLeftShift ? ISD::SHL : ISD::SRL, DL, WideVT, Vec,
- DAG.getConstant(ShiftAmt, DL, WideVT));
- return DAG.getBitcast(VT, Result);
- }
- }
-
// Check for slide-with-zeros pattern before EXT (slide is also valid EXT)
{
unsigned ShiftAmount;
bool IsRightShift;
if (SDValue DataVec = isSlideWithZerosMask(ShuffleMask, VT, V1, V2,
ShiftAmount, IsRightShift)) {
- MVT ShiftVT = VT.getSizeInBits() == 64 ? MVT::v1i64 : MVT::v2i64;
+ MVT ShiftVT;
+ if (ShiftAmount >= 32)
+ ShiftVT = VT.getSizeInBits() == 64 ? MVT::v1i64 : MVT::v2i64;
+ else if (ShiftAmount >= 16)
+ ShiftVT = VT.getSizeInBits() == 64 ? MVT::v2i32 : MVT::v4i32;
+ else
+ ShiftVT = VT.getSizeInBits() == 64 ? MVT::v4i16 : MVT::v8i16;
SDValue Vec = DAG.getNode(AArch64ISD::NVCAST, DL, ShiftVT, DataVec);
SDValue ShiftAmt = DAG.getTargetConstant(ShiftAmount, DL, MVT::i32);
More information about the llvm-commits
mailing list