[llvm] 15fe4ef - [AArch64] Fold zero-interleave shuffle into vector shift left (#210793)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 4 03:06:59 PDT 2026
Author: Deepak Shirke
Date: 2026-08-04T10:06:53Z
New Revision: 15fe4ef193d37bc108b27da566b7a96111fa3e4e
URL: https://github.com/llvm/llvm-project/commit/15fe4ef193d37bc108b27da566b7a96111fa3e4e
DIFF: https://github.com/llvm/llvm-project/commit/15fe4ef193d37bc108b27da566b7a96111fa3e4e.diff
LOG: [AArch64] Fold zero-interleave shuffle into vector shift left (#210793)
A shuffle mask that interleaves zeros between every other byte element
is equivalent to a vector shift left on a wider element type. This
avoids generating a `tbl` instruction with a constant mask loaded from
memory, replacing it with a single `shl` instruction.
Before:
```asm
adrp x8, .LCPI0_0
ldr q1, [x8, :lo12:.LCPI0_0]
tbl v0.16b, { v0.16b }, v1.16b
```
After:
```asm
shl v0.8h, v0.8h, #8
```
Both `<16 x i8>` and `<8 x i8>` vectors are handled. The canonicalized
form where zeros appear in either the first or second shuffle operand is
also handled.
Fixes #107287
Added:
llvm/test/CodeGen/AArch64/shuffle-zero-interleave-to-shl.ll
Modified:
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 51be0e66b19b0..3321da411aa95 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -15145,7 +15145,8 @@ static unsigned checkLaneSlide(ArrayRef<int> Mask, unsigned LaneStart,
static SDValue isSlideWithZerosMask(ArrayRef<int> M, EVT VT, SDValue V1,
SDValue V2, unsigned &ShiftAmount,
- bool &IsRightShift) {
+ bool &IsRightShift,
+ unsigned &MatchedLaneSize) {
unsigned VTSize = VT.getSizeInBits();
if (VTSize != 64 && VTSize != 128)
return SDValue();
@@ -15168,31 +15169,41 @@ static SDValue isSlideWithZerosMask(ArrayRef<int> M, EVT VT, SDValue V1,
DataVec = V2;
}
- // For 64-bit vectors, check single lane
- // For 128-bit vectors, check both 64-bit lanes have same slide
- unsigned LaneElts = 64 / EltSize;
- unsigned NumLanes = VTSize / 64;
+ // Try lane sizes 64, 32, 16 bits.
+ // For each lane size, check all lanes have the same slide pattern.
+ for (unsigned LaneSize : {64u, 32u, 16u}) {
+ if (LaneSize < EltSize * 2)
+ break; // need at least 2 elements per lane
+ unsigned LaneElts = LaneSize / EltSize;
+ unsigned NumLanes = VTSize / LaneSize;
- bool FirstIsLeftSlide;
- unsigned FirstSlideAmt =
- checkLaneSlide(Mask, 0, LaneElts, NumElts, FirstIsLeftSlide);
- if (FirstSlideAmt == 0)
- return SDValue();
+ bool FirstIsLeftSlide;
+ unsigned FirstSlideAmt =
+ checkLaneSlide(Mask, 0, LaneElts, NumElts, FirstIsLeftSlide);
+ if (FirstSlideAmt == 0)
+ continue;
- // For 128-bit, verify second lane matches
- if (NumLanes == 2) {
- bool SecondIsLeftSlide;
- unsigned SecondSlideAmt =
- checkLaneSlide(Mask, LaneElts, LaneElts, NumElts, SecondIsLeftSlide);
- if (SecondSlideAmt != FirstSlideAmt ||
- SecondIsLeftSlide != FirstIsLeftSlide)
- return SDValue();
- }
+ // Verify all lanes match
+ bool AllMatch = true;
+ for (unsigned Lane = 1; Lane < NumLanes; Lane++) {
+ bool IsLeftSlide;
+ unsigned SlideAmt =
+ checkLaneSlide(Mask, Lane * LaneElts, LaneElts, NumElts, IsLeftSlide);
+ if (SlideAmt != FirstSlideAmt || IsLeftSlide != FirstIsLeftSlide) {
+ AllMatch = false;
+ break;
+ }
+ }
+ if (!AllMatch)
+ continue;
- ShiftAmount = FirstSlideAmt * EltSize;
- IsRightShift = FirstIsLeftSlide; // left slide = right shift in bits
- if (ShiftAmount > 0 && ShiftAmount < 64)
- return DataVec;
+ ShiftAmount = FirstSlideAmt * EltSize;
+ IsRightShift = FirstIsLeftSlide;
+ if (ShiftAmount > 0 && ShiftAmount < LaneSize) {
+ MatchedLaneSize = LaneSize;
+ return DataVec;
+ }
+ }
return SDValue();
}
@@ -15953,9 +15964,12 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
{
unsigned ShiftAmount;
bool IsRightShift;
- if (SDValue DataVec = isSlideWithZerosMask(ShuffleMask, VT, V1, V2,
- ShiftAmount, IsRightShift)) {
- MVT ShiftVT = VT.getSizeInBits() == 64 ? MVT::v1i64 : MVT::v2i64;
+ unsigned MatchedLaneSize;
+ if (SDValue DataVec =
+ isSlideWithZerosMask(ShuffleMask, VT, V1, V2, ShiftAmount,
+ IsRightShift, MatchedLaneSize)) {
+ MVT ShiftVT = MVT::getVectorVT(MVT::getIntegerVT(MatchedLaneSize),
+ VT.getSizeInBits() / MatchedLaneSize);
SDValue Vec = DAG.getNode(AArch64ISD::NVCAST, DL, ShiftVT, DataVec);
SDValue ShiftAmt = DAG.getTargetConstant(ShiftAmount, DL, MVT::i32);
diff --git a/llvm/test/CodeGen/AArch64/shuffle-zero-interleave-to-shl.ll b/llvm/test/CodeGen/AArch64/shuffle-zero-interleave-to-shl.ll
new file mode 100644
index 0000000000000..db508d9fb256d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/shuffle-zero-interleave-to-shl.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc < %s -mtriple=aarch64 | FileCheck %s
+
+; Shuffle patterns that interleave zeros between bytes are equivalent to
+; vector shift left on wider element type. These should use shl
+; instead of tbl with a constant mask loaded from memory.
+
+; SHL pattern: zeros in even positions, data in odd positions
+define <16 x i8> @shl_v16i8_data_first(<16 x i8> %v) {
+; CHECK-LABEL: shl_v16i8_data_first:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.8h, v0.8h, #8
+; CHECK-NEXT: ret
+ %r = shufflevector <16 x i8> %v, <16 x i8> zeroinitializer, <16 x i32> <i32 16, i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14>
+ ret <16 x i8> %r
+}
+
+; SHL pattern: zeros in first vector, data in second vector
+define <16 x i8> @shl_v16i8_zeros_first(<16 x i8> %v) {
+; CHECK-LABEL: shl_v16i8_zeros_first:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.8h, v0.8h, #8
+; CHECK-NEXT: ret
+ %r = shufflevector <16 x i8> zeroinitializer, <16 x i8> %v, <16 x i32> <i32 0, i32 16, i32 0, i32 18, i32 0, i32 20, i32 0, i32 22, i32 0, i32 24, i32 0, i32 26, i32 0, i32 28, i32 0, i32 30>
+ ret <16 x i8> %r
+}
+
+; v8i8 case
+define <8 x i8> @shl_v8i8(<8 x i8> %v) {
+; CHECK-LABEL: shl_v8i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-NEXT: ret
+ %r = shufflevector <8 x i8> %v, <8 x i8> zeroinitializer, <8 x i32> <i32 8, i32 0, i32 8, i32 2, i32 8, i32 4, i32 8, i32 6>
+ ret <8 x i8> %r
+}
+
+; i16 element case: <8 x i16> zero-interleave --> shl <4 x i32>, #16
+define <8 x i16> @shl_v8i16(<8 x i16> %v) {
+; CHECK-LABEL: shl_v8i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.4s, v0.4s, #16
+; CHECK-NEXT: ret
+ %r = shufflevector <8 x i16> %v, <8 x i16> zeroinitializer, <8 x i32> <i32 8, i32 0, i32 8, i32 2, i32 8, i32 4, i32 8, i32 6>
+ ret <8 x i16> %r
+}
+
+; i32 element case: <4 x i32> zero-interleave --> shl <2 x i64>, #32
+define <4 x i32> @shl_v4i32(<4 x i32> %v) {
+; CHECK-LABEL: shl_v4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.2d, v0.2d, #32
+; CHECK-NEXT: ret
+ %r = shufflevector <4 x i32> %v, <4 x i32> zeroinitializer, <4 x i32> <i32 4, i32 0, i32 4, i32 2>
+ ret <4 x i32> %r
+}
More information about the llvm-commits
mailing list