[llvm] 9bbed74 - [AArch64] Optimize vector slide shuffles with zeros to use shift instructions (#185170)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 12 08:55:13 PDT 2026
Author: dibrinsofor
Date: 2026-06-12T16:55:08+01:00
New Revision: 9bbed741e260753de029fcf7b22844fe23362a39
URL: https://github.com/llvm/llvm-project/commit/9bbed741e260753de029fcf7b22844fe23362a39
DIFF: https://github.com/llvm/llvm-project/commit/9bbed741e260753de029fcf7b22844fe23362a39.diff
LOG: [AArch64] Optimize vector slide shuffles with zeros to use shift instructions (#185170)
We currently emit `movi`+`ext` instructions when generating code for
shuffle slides of a 64-bit vector left/right and fill it with zeros.
This patch optimizes these patterns to use a single `ushr`/`shl`
instruction instead.
Example:
```llvm
define <8 x i8> @slide_left(<8 x i8> %v) {
%r = shufflevector <8 x i8> %v, <8 x i8> zeroinitializer,
<8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
ret <8 x i8> %r
}
```
Before, we generate:
```
movi v1.2d, #0
ext v0.8b, v0.8b, v1.8b, #1
```
Now:
```
ushr d0, d0, #8
```
Fixes: #183398
Alive2 proof: https://alive2.llvm.org/ce/z/QaW5CQ
---------
Signed-off-by: Dibri Nsofor <dibrinsofor at gmail.com>
Added:
llvm/test/CodeGen/AArch64/shuffle-slide-to-shift.ll
Modified:
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/test/CodeGen/AArch64/ext-narrow-index.ll
llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index b5e82333e0801..faf102f300659 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -14570,6 +14570,119 @@ static bool isEXTMask(ArrayRef<int> M, EVT VT, bool &ReverseEXT,
return true;
}
+/// Flag slide shuffle patterns where one operand is zeros.
+/// Left slide: shufflevector %v, zeros, <1,2,3,...> -> ushr
+/// Right slide: shufflevector zeros, %v, <N-1,N,N+1,...> -> shl
+/// Check if a single 64-bit lane has a valid slide pattern.
+/// LaneStart: first element index of this lane in the full vector
+/// LaneElts: number of elements in the lane
+/// Returns slide amount in elements, or 0 if not a valid slide.
+static unsigned checkLaneSlide(ArrayRef<int> Mask, unsigned LaneStart,
+ unsigned LaneElts, unsigned NumElts,
+ bool &IsLeftSlide) {
+ // Check for left slide: <k, k+1, ..., LaneElts-1, zero, ...>
+ // where k > 0 and elements stay within lane
+ int FirstIdx = Mask[LaneStart];
+ if (FirstIdx > (int)LaneStart && FirstIdx < (int)(LaneStart + LaneElts)) {
+ unsigned SlideAmt = FirstIdx - LaneStart;
+ for (unsigned i = 0; i < LaneElts; ++i) {
+ int MaskIdx = Mask[LaneStart + i];
+ if (MaskIdx < 0)
+ continue;
+ if (i < LaneElts - SlideAmt) {
+ // Data element: must be consecutive within lane
+ if (MaskIdx != (int)(LaneStart + SlideAmt + i))
+ return 0;
+ } else {
+ // Zero element: any index >= NumElts is fine (all from V2 which is
+ // zeros)
+ if (MaskIdx < (int)NumElts)
+ return 0;
+ }
+ }
+ IsLeftSlide = true;
+ return SlideAmt;
+ }
+
+ // Check for right slide: <zero, ..., 0, 1, ...>
+ // where zeros come first, then consecutive from lane start
+ if (Mask[LaneStart] >= (int)NumElts || Mask[LaneStart] < 0) {
+ unsigned ZeroCount = 0;
+ for (unsigned i = 0; i < LaneElts; ++i) {
+ int MaskIdx = Mask[LaneStart + i];
+ if (MaskIdx >= 0 && MaskIdx < (int)NumElts)
+ break;
+ ZeroCount++;
+ }
+ if (ZeroCount > 0 && ZeroCount < LaneElts) {
+ for (unsigned i = ZeroCount; i < LaneElts; ++i) {
+ int MaskIdx = Mask[LaneStart + i];
+ if (MaskIdx < 0)
+ continue;
+ if (MaskIdx != (int)(LaneStart + i - ZeroCount))
+ return 0;
+ }
+ IsLeftSlide = false;
+ return ZeroCount;
+ }
+ }
+
+ return 0;
+}
+
+static SDValue isSlideWithZerosMask(ArrayRef<int> M, EVT VT, SDValue V1,
+ SDValue V2, unsigned &ShiftAmount,
+ bool &IsRightShift) {
+ unsigned VTSize = VT.getSizeInBits();
+ if (VTSize != 64 && VTSize != 128)
+ return SDValue();
+
+ unsigned NumElts = VT.getVectorNumElements();
+ unsigned EltSize = VT.getScalarSizeInBits();
+
+ bool V1IsZeros = ISD::isBuildVectorAllZeros(V1.getNode());
+ bool V2IsZeros = ISD::isBuildVectorAllZeros(V2.getNode());
+
+ // Exactly one operand must be zeros
+ if (V1IsZeros == V2IsZeros)
+ return SDValue();
+
+ // Canonicalize so V2 is zeros
+ SmallVector<int, 16> Mask(M.begin(), M.end());
+ SDValue DataVec = V1;
+ if (V1IsZeros) {
+ ShuffleVectorSDNode::commuteMask(Mask);
+ DataVec = V2;
+ }
+
+ // For 64-bit vectors, check single lane
+ // For 128-bit vectors, check both 64-bit lanes have same slide
+ unsigned LaneElts = 64 / EltSize;
+ unsigned NumLanes = VTSize / 64;
+
+ bool FirstIsLeftSlide;
+ unsigned FirstSlideAmt =
+ checkLaneSlide(Mask, 0, LaneElts, NumElts, FirstIsLeftSlide);
+ if (FirstSlideAmt == 0)
+ return SDValue();
+
+ // For 128-bit, verify second lane matches
+ if (NumLanes == 2) {
+ bool SecondIsLeftSlide;
+ unsigned SecondSlideAmt =
+ checkLaneSlide(Mask, LaneElts, LaneElts, NumElts, SecondIsLeftSlide);
+ if (SecondSlideAmt != FirstSlideAmt ||
+ SecondIsLeftSlide != FirstIsLeftSlide)
+ return SDValue();
+ }
+
+ ShiftAmount = FirstSlideAmt * EltSize;
+ IsRightShift = FirstIsLeftSlide; // left slide = right shift in bits
+ if (ShiftAmount > 0 && ShiftAmount < 64)
+ return DataVec;
+ return SDValue();
+}
+
// Check if an EXT instruction can handle the shuffle mask when one source is a
// splat. This matches shuffles where the splat occupies either a prefix or a
// suffix and the remaining lanes are a contiguous slice from the non-splat
@@ -15325,6 +15438,23 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
DAG.getConstant(8, DL, MVT::i32));
}
+ // Check for slide-with-zeros pattern before EXT (slide is also valid EXT)
+ {
+ unsigned ShiftAmount;
+ bool IsRightShift;
+ if (SDValue DataVec = isSlideWithZerosMask(ShuffleMask, VT, V1, V2,
+ ShiftAmount, IsRightShift)) {
+ MVT ShiftVT = VT.getSizeInBits() == 64 ? MVT::v1i64 : MVT::v2i64;
+ SDValue Vec = DAG.getNode(AArch64ISD::NVCAST, DL, ShiftVT, DataVec);
+
+ SDValue ShiftAmt = DAG.getTargetConstant(ShiftAmount, DL, MVT::i32);
+ unsigned Opc = IsRightShift ? AArch64ISD::VLSHR : AArch64ISD::VSHL;
+ SDValue Shifted = DAG.getNode(Opc, DL, ShiftVT, Vec, ShiftAmt);
+
+ return DAG.getNode(AArch64ISD::NVCAST, DL, VT, Shifted);
+ }
+ }
+
bool IsSplat1 =
V1.getValueType() == VT && DAG.isSplatValue(V1, /*AllowUndefs=*/false);
bool IsSplat2 =
diff --git a/llvm/test/CodeGen/AArch64/ext-narrow-index.ll b/llvm/test/CodeGen/AArch64/ext-narrow-index.ll
index e275bd1fb9241..a83cbb08f1a8e 100644
--- a/llvm/test/CodeGen/AArch64/ext-narrow-index.ll
+++ b/llvm/test/CodeGen/AArch64/ext-narrow-index.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc < %s -global-isel -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-GISEL
@@ -218,12 +218,18 @@ entry:
}
define <8 x i8> @i8_zero_off15(<16 x i8> %arg1) {
-; CHECK-LABEL: i8_zero_off15:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: ext v0.16b, v0.16b, v1.16b, #15
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: i8_zero_off15:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: mov d0, v0.d[1]
+; CHECK-SD-NEXT: ushr d0, d0, #56
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GISEL-LABEL: i8_zero_off15:
+; CHECK-GISEL: // %bb.0: // %entry
+; CHECK-GISEL-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GISEL-NEXT: ext v0.16b, v0.16b, v1.16b, #15
+; CHECK-GISEL-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GISEL-NEXT: ret
entry:
%shuffle = shufflevector <16 x i8> %arg1, <16 x i8> zeroinitializer, <8 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22>
ret <8 x i8> %shuffle
@@ -269,12 +275,18 @@ entry:
}
define <4 x i16> @i16_zero_off7(<8 x i16> %arg1) {
-; CHECK-LABEL: i16_zero_off7:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: ext v0.16b, v0.16b, v1.16b, #14
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: i16_zero_off7:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: mov d0, v0.d[1]
+; CHECK-SD-NEXT: ushr d0, d0, #48
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GISEL-LABEL: i16_zero_off7:
+; CHECK-GISEL: // %bb.0: // %entry
+; CHECK-GISEL-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GISEL-NEXT: ext v0.16b, v0.16b, v1.16b, #14
+; CHECK-GISEL-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GISEL-NEXT: ret
entry:
%shuffle = shufflevector <8 x i16> %arg1, <8 x i16> zeroinitializer, <4 x i32> <i32 7, i32 8, i32 9, i32 10>
ret <4 x i16> %shuffle
@@ -313,12 +325,18 @@ entry:
}
define <2 x i32> @i32_zero_off3(<4 x i32> %arg1) {
-; CHECK-LABEL: i32_zero_off3:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: ext v0.16b, v0.16b, v1.16b, #12
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: i32_zero_off3:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: mov d0, v0.d[1]
+; CHECK-SD-NEXT: ushr d0, d0, #32
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GISEL-LABEL: i32_zero_off3:
+; CHECK-GISEL: // %bb.0: // %entry
+; CHECK-GISEL-NEXT: movi v1.2d, #0000000000000000
+; CHECK-GISEL-NEXT: ext v0.16b, v0.16b, v1.16b, #12
+; CHECK-GISEL-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GISEL-NEXT: ret
entry:
%shuffle = shufflevector <4 x i32> %arg1, <4 x i32> zeroinitializer, <2 x i32> <i32 3, i32 4>
ret <2 x i32> %shuffle
diff --git a/llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll b/llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll
index 8eac8f85126f1..28603ae8b5cef 100644
--- a/llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/fp-conversion-to-tbl.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -o - %s | FileCheck %s
target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
@@ -724,75 +724,69 @@ define void @uitofp_ld4_v32i16_to_v8f64(ptr nocapture noundef readonly %x, ptr n
; CHECK-NEXT: Lloh33:
; CHECK-NEXT: ldr q0, [x8, lCPI11_0 at PAGEOFF]
; CHECK-NEXT: Lloh34:
-; CHECK-NEXT: adrp x8, lCPI11_3 at PAGE
-; CHECK-NEXT: Lloh35:
; CHECK-NEXT: ldr q1, [x9, lCPI11_1 at PAGEOFF]
-; CHECK-NEXT: Lloh36:
+; CHECK-NEXT: Lloh35:
; CHECK-NEXT: ldr q2, [x10, lCPI11_2 at PAGEOFF]
-; CHECK-NEXT: Lloh37:
-; CHECK-NEXT: ldr q3, [x8, lCPI11_3 at PAGEOFF]
; CHECK-NEXT: mov x8, xzr
; CHECK-NEXT: LBB11_1: ; %vector.body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: add x9, x0, x8
-; CHECK-NEXT: ldp q5, q4, [x9, #32]
-; CHECK-NEXT: ldp q7, q6, [x9]
+; CHECK-NEXT: ldp q3, q4, [x9, #32]
+; CHECK-NEXT: ldp q5, q6, [x9]
; CHECK-NEXT: add x9, x1, x8
; CHECK-NEXT: add x8, x8, #64
-; CHECK-NEXT: tbl.16b v16, { v4 }, v0
-; CHECK-NEXT: tbl.16b v17, { v5 }, v0
+; CHECK-NEXT: tbl.16b v7, { v4 }, v0
+; CHECK-NEXT: tbl.16b v16, { v3 }, v0
; CHECK-NEXT: tbl.16b v21, { v4 }, v1
-; CHECK-NEXT: tbl.16b v18, { v6 }, v0
-; CHECK-NEXT: tbl.16b v19, { v7 }, v0
-; CHECK-NEXT: tbl.16b v20, { v7 }, v1
-; CHECK-NEXT: tbl.16b v22, { v5 }, v1
-; CHECK-NEXT: tbl.16b v23, { v5 }, v2
+; CHECK-NEXT: tbl.16b v17, { v6 }, v0
+; CHECK-NEXT: tbl.16b v18, { v5 }, v0
+; CHECK-NEXT: tbl.16b v19, { v6 }, v1
+; CHECK-NEXT: tbl.16b v20, { v5 }, v1
+; CHECK-NEXT: tbl.16b v22, { v3 }, v1
+; CHECK-NEXT: tbl.16b v23, { v3 }, v2
; CHECK-NEXT: tbl.16b v24, { v4 }, v2
-; CHECK-NEXT: tbl.16b v25, { v7 }, v2
-; CHECK-NEXT: tbl.16b v5, { v5 }, v3
-; CHECK-NEXT: tbl.16b v4, { v4 }, v3
-; CHECK-NEXT: tbl.16b v7, { v7 }, v3
-; CHECK-NEXT: tbl.16b v26, { v6 }, v1
-; CHECK-NEXT: tbl.16b v27, { v6 }, v2
-; CHECK-NEXT: tbl.16b v6, { v6 }, v3
-; CHECK-NEXT: ucvtf.2d v17, v17
+; CHECK-NEXT: tbl.16b v25, { v5 }, v2
+; CHECK-NEXT: tbl.16b v26, { v6 }, v2
+; CHECK-NEXT: ushr.2d v3, v3, #48
+; CHECK-NEXT: ushr.2d v4, v4, #48
+; CHECK-NEXT: ushr.2d v5, v5, #48
+; CHECK-NEXT: ushr.2d v6, v6, #48
; CHECK-NEXT: ucvtf.2d v16, v16
-; CHECK-NEXT: ucvtf.2d v19, v19
+; CHECK-NEXT: ucvtf.2d v7, v7
; CHECK-NEXT: ucvtf.2d v18, v18
+; CHECK-NEXT: ucvtf.2d v17, v17
; CHECK-NEXT: ucvtf.2d v22, v22
; CHECK-NEXT: ucvtf.2d v23, v23
-; CHECK-NEXT: ucvtf.2d v5, v5
+; CHECK-NEXT: ucvtf.2d v3, v3
; CHECK-NEXT: ucvtf.2d v21, v21
; CHECK-NEXT: ucvtf.2d v24, v24
; CHECK-NEXT: ucvtf.2d v4, v4
; CHECK-NEXT: cmp x8, #2, lsl #12 ; =8192
; CHECK-NEXT: ucvtf.2d v20, v20
; CHECK-NEXT: ucvtf.2d v25, v25
-; CHECK-NEXT: ucvtf.2d v7, v7
+; CHECK-NEXT: ucvtf.2d v5, v5
+; CHECK-NEXT: ucvtf.2d v19, v19
; CHECK-NEXT: ucvtf.2d v26, v26
-; CHECK-NEXT: ucvtf.2d v27, v27
; CHECK-NEXT: ucvtf.2d v6, v6
-; CHECK-NEXT: fadd.2d v17, v22, v17
-; CHECK-NEXT: fadd.2d v5, v23, v5
-; CHECK-NEXT: fadd.2d v16, v21, v16
+; CHECK-NEXT: fadd.2d v16, v22, v16
+; CHECK-NEXT: fadd.2d v3, v23, v3
+; CHECK-NEXT: fadd.2d v7, v21, v7
; CHECK-NEXT: fadd.2d v4, v24, v4
-; CHECK-NEXT: fadd.2d v19, v20, v19
-; CHECK-NEXT: fadd.2d v7, v25, v7
-; CHECK-NEXT: fadd.2d v18, v26, v18
-; CHECK-NEXT: fadd.2d v6, v27, v6
-; CHECK-NEXT: fadd.2d v5, v17, v5
-; CHECK-NEXT: fadd.2d v4, v16, v4
-; CHECK-NEXT: fadd.2d v7, v19, v7
-; CHECK-NEXT: fadd.2d v6, v18, v6
-; CHECK-NEXT: stp q5, q4, [x9, #32]
-; CHECK-NEXT: stp q7, q6, [x9]
+; CHECK-NEXT: fadd.2d v18, v20, v18
+; CHECK-NEXT: fadd.2d v5, v25, v5
+; CHECK-NEXT: fadd.2d v17, v19, v17
+; CHECK-NEXT: fadd.2d v6, v26, v6
+; CHECK-NEXT: fadd.2d v3, v16, v3
+; CHECK-NEXT: fadd.2d v4, v7, v4
+; CHECK-NEXT: fadd.2d v5, v18, v5
+; CHECK-NEXT: fadd.2d v6, v17, v6
+; CHECK-NEXT: stp q3, q4, [x9, #32]
+; CHECK-NEXT: stp q5, q6, [x9]
; CHECK-NEXT: b.ne LBB11_1
; CHECK-NEXT: ; %bb.2: ; %for.cond.cleanup
; CHECK-NEXT: ret
-; CHECK-NEXT: .loh AdrpLdr Lloh34, Lloh37
-; CHECK-NEXT: .loh AdrpLdr Lloh32, Lloh36
-; CHECK-NEXT: .loh AdrpLdr Lloh31, Lloh35
-; CHECK-NEXT: .loh AdrpAdrp Lloh30, Lloh34
+; CHECK-NEXT: .loh AdrpLdr Lloh32, Lloh35
+; CHECK-NEXT: .loh AdrpLdr Lloh31, Lloh34
; CHECK-NEXT: .loh AdrpLdr Lloh30, Lloh33
entry:
br label %vector.body
diff --git a/llvm/test/CodeGen/AArch64/shuffle-slide-to-shift.ll b/llvm/test/CodeGen/AArch64/shuffle-slide-to-shift.ll
new file mode 100644
index 0000000000000..fd52fd26a6230
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/shuffle-slide-to-shift.ll
@@ -0,0 +1,223 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64 < %s | FileCheck %s
+
+
+; Left slide: {v[0]..v[7]} => {v[1]..v[7], 0}
+define <8 x i8> @slide_left_1(<8 x i8> %v) {
+; CHECK-LABEL: slide_left_1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushr d0, d0, #8
+; CHECK-NEXT: ret
+ %r = shufflevector <8 x i8> %v, <8 x i8> zeroinitializer,
+ <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+ ret <8 x i8> %r
+}
+
+; Right slide: {v[0]..v[7]} => {0, v[0]..v[6]}
+define <8 x i8> @slide_right_1(<8 x i8> %v) {
+; CHECK-LABEL: slide_right_1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl d0, d0, #8
+; CHECK-NEXT: ret
+ %r = shufflevector <8 x i8> zeroinitializer, <8 x i8> %v,
+ <8 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
+ ret <8 x i8> %r
+}
+
+; Commuted left slide: zeros first, then data
+define <8 x i8> @slide_left_1_commuted(<8 x i8> %v) {
+; CHECK-LABEL: slide_left_1_commuted:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushr d0, d0, #8
+; CHECK-NEXT: ret
+ %r = shufflevector <8 x i8> zeroinitializer, <8 x i8> %v,
+ <8 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0>
+ ret <8 x i8> %r
+}
+
+; Commuted right slide: data first, then zeros
+define <8 x i8> @slide_right_1_commuted(<8 x i8> %v) {
+; CHECK-LABEL: slide_right_1_commuted:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl d0, d0, #8
+; CHECK-NEXT: ret
+ %r = shufflevector <8 x i8> %v, <8 x i8> zeroinitializer,
+ <8 x i32> <i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6>
+ ret <8 x i8> %r
+}
+
+; Commuted 128-bit lane-independent
+define <16 x i8> @slide_left_v16i8_lane_independent_commuted(<16 x i8> %v) {
+; CHECK-LABEL: slide_left_v16i8_lane_independent_commuted:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushr v0.2d, v0.2d, #8
+; CHECK-NEXT: ret
+ %r = shufflevector <16 x i8> zeroinitializer, <16 x i8> %v,
+ <16 x i32> <i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 0,
+ i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 8>
+ ret <16 x i8> %r
+}
+
+; 64 bit test (with poison)
+
+; Left slide with poison (from issue's Alive2 proof)
+define <8 x i8> @slide_left_poison(<8 x i8> %v) {
+; CHECK-LABEL: slide_left_poison:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushr d0, d0, #8
+; CHECK-NEXT: ret
+ %r = shufflevector <8 x i8> %v, <8 x i8> <i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>,
+ <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+ ret <8 x i8> %r
+}
+
+; Right slide with poison (from issue's Alive2 proof)
+define <8 x i8> @slide_right_poison(<8 x i8> %v) {
+; CHECK-LABEL: slide_right_poison:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl d0, d0, #8
+; CHECK-NEXT: ret
+ %r = shufflevector <8 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0>, <8 x i8> %v,
+ <8 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
+ ret <8 x i8> %r
+}
+
+; should also optimize 32-bit vectors
+
+; Left slide 32-bit: <4 x i8>
+define <4 x i8> @slide_left_v4i8(<4 x i8> %v) {
+; CHECK-LABEL: slide_left_v4i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushr d0, d0, #16
+; CHECK-NEXT: ret
+ %r = shufflevector <4 x i8> %v, <4 x i8> zeroinitializer,
+ <4 x i32> <i32 1, i32 2, i32 3, i32 4>
+ ret <4 x i8> %r
+}
+
+; Right slide 32-bit: <4 x i8>
+define <4 x i8> @slide_right_v4i8(<4 x i8> %v) {
+; CHECK-LABEL: slide_right_v4i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl d0, d0, #16
+; CHECK-NEXT: ret
+ %r = shufflevector <4 x i8> zeroinitializer, <4 x i8> %v,
+ <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+ ret <4 x i8> %r
+}
+
+; should NOT optimize 128-bit vectors
+
+; Left slide 128-bit: <16 x i8> - must use ext
+define <16 x i8> @slide_left_v16i8(<16 x i8> %v) {
+; CHECK-LABEL: slide_left_v16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: ext v0.16b, v0.16b, v1.16b, #1
+; CHECK-NEXT: ret
+ %r = shufflevector <16 x i8> %v, <16 x i8> zeroinitializer,
+ <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8,
+ i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>
+ ret <16 x i8> %r
+}
+
+; Right slide 128-bit: <16 x i8> - must use ext
+define <16 x i8> @slide_right_v16i8(<16 x i8> %v) {
+; CHECK-LABEL: slide_right_v16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: ext v0.16b, v1.16b, v0.16b, #15
+; CHECK-NEXT: ret
+ %r = shufflevector <16 x i8> zeroinitializer, <16 x i8> %v,
+ <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22,
+ i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
+ ret <16 x i8> %r
+}
+
+; 128-bit lane-independent slides (both 64-bit halves slide same amount)
+; These CAN be optimized to ushr/shl on v2i64
+
+; Left slide 128-bit lane-independent: each half slides left by 1
+define <16 x i8> @slide_left_v16i8_lane_independent(<16 x i8> %v) {
+; CHECK-LABEL: slide_left_v16i8_lane_independent:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushr v0.2d, v0.2d, #8
+; CHECK-NEXT: ret
+ %r = shufflevector <16 x i8> %v, <16 x i8> zeroinitializer,
+ <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16,
+ i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 24>
+ ret <16 x i8> %r
+}
+
+; Right slide 128-bit lane-independent: each half slides right by 1
+define <16 x i8> @slide_right_v16i8_lane_independent(<16 x i8> %v) {
+; CHECK-LABEL: slide_right_v16i8_lane_independent:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.2d, v0.2d, #8
+; CHECK-NEXT: ret
+ %r = shufflevector <16 x i8> zeroinitializer, <16 x i8> %v,
+ <16 x i32> <i32 0, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22,
+ i32 8, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
+ ret <16 x i8> %r
+}
+
+; 128-bit i16 lane-independent slide
+define <8 x i16> @slide_left_v8i16_lane_independent(<8 x i16> %v) {
+; CHECK-LABEL: slide_left_v8i16_lane_independent:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushr v0.2d, v0.2d, #16
+; CHECK-NEXT: ret
+ %r = shufflevector <8 x i16> %v, <8 x i16> zeroinitializer,
+ <8 x i32> <i32 1, i32 2, i32 3, i32 8,
+ i32 5, i32 6, i32 7, i32 12>
+ ret <8 x i16> %r
+}
+
+; 128-bit i32 lane-independent slide
+define <4 x i32> @slide_left_v4i32_lane_independent(<4 x i32> %v) {
+; CHECK-LABEL: slide_left_v4i32_lane_independent:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushr v0.2d, v0.2d, #32
+; CHECK-NEXT: ret
+ %r = shufflevector <4 x i32> %v, <4 x i32> zeroinitializer,
+ <4 x i32> <i32 1, i32 4, i32 3, i32 6>
+ ret <4 x i32> %r
+}
+
+; Negative: halves slide
diff erent amounts (should NOT optimize to shift)
+define <16 x i8> @slide_
diff erent_amounts(<16 x i8> %v) {
+; CHECK-LABEL: slide_
diff erent_amounts:
+; CHECK: // %bb.0:
+; CHECK-NEXT: adrp x8, .LCPI15_0
+; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI15_0]
+; CHECK-NEXT: tbl v0.16b, { v0.16b }, v1.16b
+; CHECK-NEXT: ret
+ %r = shufflevector <16 x i8> %v, <16 x i8> zeroinitializer,
+ <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16,
+ i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 24, i32 25>
+ ret <16 x i8> %r
+}
+
+; Slide by max (N-1 elements)
+
+; Left slide by 7 (max for v8i8)
+define <8 x i8> @slide_left_max(<8 x i8> %v) {
+; CHECK-LABEL: slide_left_max:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushr d0, d0, #56
+; CHECK-NEXT: ret
+ %r = shufflevector <8 x i8> %v, <8 x i8> zeroinitializer,
+ <8 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
+ ret <8 x i8> %r
+}
+
+; Right slide by 7 (max for v8i8): [v0..v7] => [0,0,0,0,0,0,0,v0]
+define <8 x i8> @slide_right_max(<8 x i8> %v) {
+; CHECK-LABEL: slide_right_max:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl d0, d0, #56
+; CHECK-NEXT: ret
+ %r = shufflevector <8 x i8> zeroinitializer, <8 x i8> %v,
+ <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 8>
+ ret <8 x i8> %r
+}
More information about the llvm-commits
mailing list