[llvm-branch-commits] [llvm] [SLP] Price gathered lanes of vectorized entries as a shuffle (PR #240136)
Dmitry Sidorov via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Nov 8 05:13:37 PST 2026
https://github.com/MrSidims created https://github.com/llvm/llvm-project/pull/240136
A gathered lane already vectorized in another entry is moved out of
that vector with a shuffle, which a zero insert price misses. Add the
shuffle cost of these lanes when their insert price is zero.
>From 470677065d6fe44a0f290e010592d3f211399e54 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sat, 7 Nov 2026 22:50:22 +0100
Subject: [PATCH] [SLP] Price gathered lanes of vectorized entries as a shuffle
A gathered lane already vectorized in another entry is moved out of
that vector with a shuffle, which a zero insert price misses. Add the
shuffle cost of these lanes when their insert price is zero.
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 20 ++++++
.../gathered-lane-of-vectorized-loads.ll | 13 ++--
.../AArch64/loop-accumulator-reduction.ll | 3 +-
...recalc-copyable-operand-deps-direct-use.ll | 67 ++++++++++---------
.../trimmed-subtree-schedule-operands.ll | 29 ++++----
.../AMDGPU/gathered-load-strided-picks.ll | 2 +-
...gathered-lane-of-vectorized-entry-zve32.ll | 34 ++++++++++
.../X86/gathered-shuffle-resized.ll | 13 ++--
.../absorbed-lane-shuffled-operand.ll | 3 +-
.../extract-many-users-buildvector.ll | 43 ++++++------
...hered-consecutive-loads-different-types.ll | 8 +--
.../gathered-swapped-vectorized-lanes.ll | 26 +++----
.../resized-alt-shuffle-after-minbw.ll | 12 ++--
13 files changed, 166 insertions(+), 107 deletions(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/RISCV/gathered-lane-of-vectorized-entry-zve32.ll
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index fbef70d8aff522..9c0ceeb13f93ef 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -21932,6 +21932,26 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
getScalarizationOverhead(*TTI, Opts.slp_revec, VL.front()->getType(),
MaskVecTy, DemandedElts, /*Insert=*/true,
/*Extract=*/false, CostKind);
+ // A lane taken from another vectorized entry needs a shuffle, which a zero
+ // insert price misses.
+ APInt VectorizedElts = APInt::getZero(SubMask.size());
+ SmallVector<int> VectorizedMask(SubMask.size(), PoisonMaskElem);
+ for (auto [I, Idx] : enumerate(SubMask)) {
+ if (Idx == PoisonMaskElem || !isVectorized(VL[I]))
+ continue;
+ VectorizedElts.setBit(I);
+ VectorizedMask[I] = Idx;
+ }
+ if (!VectorizedElts.isZero() &&
+ getScalarizationOverhead(*TTI, Opts.slp_revec, VL.front()->getType(),
+ MaskVecTy, VectorizedElts, /*Insert=*/true,
+ /*Extract=*/false,
+ CostKind) == TTI::TCC_Free) {
+ InstructionCost VectorizedShuffleCost =
+ GetShuffleCost(VectorizedMask, Entries, VecTy);
+ if (VectorizedShuffleCost.isValid())
+ BuildVectorCost += VectorizedShuffleCost;
+ }
const TreeEntry *BestEntry = nullptr;
auto MaskSlice = MutableArrayRef(Mask).slice(MaskBase, VL.size());
if (FirstShuffleCost < ShuffleCost) {
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/gathered-lane-of-vectorized-loads.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/gathered-lane-of-vectorized-loads.ll
index f365aa329cb406..e80afa9cd12cc1 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/gathered-lane-of-vectorized-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/gathered-lane-of-vectorized-loads.ll
@@ -6,11 +6,14 @@
define void @load_lane_and_arg(ptr %a, double %x, ptr %out) {
; CHECK-LABEL: define void @load_lane_and_arg(
; CHECK-SAME: ptr [[A:%.*]], double [[X:%.*]], ptr [[OUT:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[A]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> poison, double [[X]], i64 1
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> [[TMP1]], <2 x i32> <i32 3, i32 1>
-; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: store <2 x double> [[TMP4]], ptr [[OUT]], align 8
+; CHECK-NEXT: [[P1:%.*]] = getelementptr double, ptr [[A]], i64 1
+; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[A]], align 8
+; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[P1]], align 8
+; CHECK-NEXT: [[M0:%.*]] = fadd double [[A0]], [[A1]]
+; CHECK-NEXT: [[M1:%.*]] = fadd double [[A1]], [[X]]
+; CHECK-NEXT: [[O1:%.*]] = getelementptr double, ptr [[OUT]], i64 1
+; CHECK-NEXT: store double [[M0]], ptr [[OUT]], align 8
+; CHECK-NEXT: store double [[M1]], ptr [[O1]], align 8
; CHECK-NEXT: ret void
;
%p1 = getelementptr double, ptr %a, i64 1
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
index e1a4cee87e5089..983c568fb01d25 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
@@ -50,8 +50,7 @@ define double @loop_acc_fadd(ptr %p, i64 %n) {
; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> [[TMP16]], double [[C8]], i64 2
; CHECK-NEXT: [[TMP18:%.*]] = fmul reassoc nnan ninf arcp contract afn <4 x double> [[TMP14]], [[TMP17]]
; CHECK-NEXT: [[TMP19:%.*]] = fadd reassoc nnan ninf nsz arcp afn <4 x double> [[TMP12]], [[TMP18]]
-; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <4 x double> [[TMP19]], <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, <4 x i32> <i32 0, i32 1, i32 poison, i32 7>
-; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> [[TMP20]], <4 x double> [[TMP19]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> [[TMP19]], <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
; CHECK-NEXT: [[TMP22:%.*]] = fmul <4 x double> [[TMP19]], [[TMP21]]
; CHECK-NEXT: [[SLPRDX_ACC1]] = fadd reassoc nnan ninf nsz arcp afn <4 x double> [[SLPRDX_ACC]], [[TMP22]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-operand-deps-direct-use.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-operand-deps-direct-use.ll
index e28b3eed83900e..ef1dd3ee4ca0a3 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-operand-deps-direct-use.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-operand-deps-direct-use.ll
@@ -9,50 +9,53 @@ define double @test(ptr %0, double %1) {
; CHECK-NEXT: [[TMP3:%.*]] = fsub double 0.000000e+00, [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = fmul double [[TMP3]], 0.000000e+00
; CHECK-NEXT: store double [[TMP4]], ptr null, align 8
-; CHECK-NEXT: [[TMP5:%.*]] = fmul double 0.000000e+00, 0.000000e+00
-; CHECK-NEXT: [[TMP6:%.*]] = fmul double [[TMP3]], 0.000000e+00
-; CHECK-NEXT: [[TMP7:%.*]] = fmul double [[TMP6]], [[TMP1]]
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x double> <double poison, double 0.000000e+00>, double [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x double> [[TMP5]], zeroinitializer
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <2 x double> poison, double [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x double> [[TMP17]], double [[TMP3]], i64 0
+; CHECK-NEXT: [[TMP20:%.*]] = fmul <2 x double> [[TMP8]], zeroinitializer
+; CHECK-NEXT: [[TMP21:%.*]] = fadd <2 x double> [[TMP8]], zeroinitializer
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x double> [[TMP20]], <2 x double> [[TMP21]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP26:%.*]] = shufflevector <2 x double> [[TMP11]], <2 x double> <double poison, double 0.000000e+00>, <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP13:%.*]] = fmul <2 x double> [[TMP26]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x double> [[TMP13]], i64 0
; CHECK-NEXT: store double [[TMP7]], ptr null, align 8
-; CHECK-NEXT: [[TMP8:%.*]] = fadd double [[TMP1]], 0.000000e+00
-; CHECK-NEXT: [[TMP9:%.*]] = fmul double [[TMP8]], 0.000000e+00
+; CHECK-NEXT: [[TMP15:%.*]] = fmul <2 x double> [[TMP11]], zeroinitializer
+; CHECK-NEXT: [[TMP16:%.*]] = fmul <2 x double> [[TMP6]], zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x double> [[TMP15]], i64 1
; CHECK-NEXT: store double [[TMP9]], ptr null, align 8
-; CHECK-NEXT: [[TMP10:%.*]] = fmul double [[TMP6]], 0.000000e+00
-; CHECK-NEXT: [[TMP11:%.*]] = fadd double 0.000000e+00, 0.000000e+00
-; CHECK-NEXT: [[TMP12:%.*]] = fmul double [[TMP11]], [[TMP1]]
+; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> <double 0.000000e+00, double poison>, <2 x i32> <i32 2, i32 0>
+; CHECK-NEXT: [[TMP19:%.*]] = fmul <2 x double> zeroinitializer, [[TMP18]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[TMP19]], i64 1
; CHECK-NEXT: store double [[TMP12]], ptr null, align 8
-; CHECK-NEXT: [[TMP13:%.*]] = fmul double [[TMP11]], 0.000000e+00
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[TMP14:%.*]] = fmul double 0.000000e+00, [[TMP4]]
-; CHECK-NEXT: [[TMP15:%.*]] = fmul double [[TMP7]], 0.000000e+00
-; CHECK-NEXT: [[TMP16:%.*]] = fadd double [[TMP15]], [[TMP4]]
-; CHECK-NEXT: [[TMP17:%.*]] = fmul double [[TMP10]], 0.000000e+00
-; CHECK-NEXT: [[TMP18:%.*]] = fadd double [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = fmul double [[TMP12]], 0.000000e+00
-; CHECK-NEXT: [[TMP20:%.*]] = fadd double [[TMP19]], 0.000000e+00
-; CHECK-NEXT: [[TMP21:%.*]] = fmul double [[TMP13]], 0.000000e+00
-; CHECK-NEXT: [[TMP22:%.*]] = fadd double [[TMP20]], [[TMP21]]
-; CHECK-NEXT: [[TMP23:%.*]] = fadd double [[TMP22]], 0.000000e+00
-; CHECK-NEXT: [[TMP24:%.*]] = fmul double [[TMP23]], 0.000000e+00
-; CHECK-NEXT: [[TMP25:%.*]] = fadd double [[TMP18]], 0.000000e+00
-; CHECK-NEXT: [[TMP26:%.*]] = fmul double [[TMP25]], 0.000000e+00
-; CHECK-NEXT: [[TMP27:%.*]] = fsub double [[TMP24]], [[TMP26]]
+; CHECK-NEXT: [[TMP22:%.*]] = fmul <2 x double> [[TMP13]], zeroinitializer
+; CHECK-NEXT: [[TMP27:%.*]] = fmul <2 x double> [[TMP15]], zeroinitializer
+; CHECK-NEXT: [[TMP24:%.*]] = fmul <2 x double> [[TMP19]], zeroinitializer
+; CHECK-NEXT: [[TMP25:%.*]] = fmul <2 x double> [[TMP16]], zeroinitializer
; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[TMP0]], i64 24
-; CHECK-NEXT: store double [[TMP27]], ptr [[TMP28]], align 8
-; CHECK-NEXT: [[TMP29:%.*]] = fmul double [[TMP5]], 0.000000e+00
-; CHECK-NEXT: [[TMP30:%.*]] = fadd double [[TMP29]], 0.000000e+00
-; CHECK-NEXT: [[TMP31:%.*]] = fmul double [[TMP9]], 0.000000e+00
-; CHECK-NEXT: [[TMP32:%.*]] = fadd double [[TMP30]], [[TMP31]]
-; CHECK-NEXT: [[TMP33:%.*]] = fadd double [[TMP32]], 0.000000e+00
-; CHECK-NEXT: [[TMP34:%.*]] = fsub double [[TMP25]], [[TMP33]]
; CHECK-NEXT: [[TMP35:%.*]] = getelementptr i8, ptr [[TMP0]], i64 32
-; CHECK-NEXT: store double [[TMP34]], ptr [[TMP35]], align 8
+; CHECK-NEXT: [[TMP34:%.*]] = insertelement <2 x double> <double poison, double 0.000000e+00>, double [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP29:%.*]] = fadd <2 x double> [[TMP22]], [[TMP34]]
+; CHECK-NEXT: [[TMP30:%.*]] = fadd <2 x double> [[TMP24]], zeroinitializer
+; CHECK-NEXT: [[TMP31:%.*]] = fadd <2 x double> [[TMP29]], [[TMP27]]
+; CHECK-NEXT: [[TMP32:%.*]] = fadd <2 x double> [[TMP30]], [[TMP25]]
+; CHECK-NEXT: [[TMP38:%.*]] = fadd <2 x double> [[TMP32]], zeroinitializer
+; CHECK-NEXT: [[TMP33:%.*]] = extractelement <2 x double> [[TMP38]], i64 1
; CHECK-NEXT: [[TMP36:%.*]] = fmul double [[TMP33]], 0.000000e+00
+; CHECK-NEXT: [[TMP47:%.*]] = fadd <2 x double> [[TMP31]], zeroinitializer
+; CHECK-NEXT: [[TMP48:%.*]] = extractelement <2 x double> [[TMP47]], i64 0
+; CHECK-NEXT: [[TMP23:%.*]] = fmul double [[TMP48]], 0.000000e+00
; CHECK-NEXT: [[TMP37:%.*]] = fsub double [[TMP36]], [[TMP23]]
-; CHECK-NEXT: [[TMP38:%.*]] = getelementptr i8, ptr [[TMP0]], i64 40
-; CHECK-NEXT: store double [[TMP37]], ptr [[TMP38]], align 8
+; CHECK-NEXT: store double [[TMP37]], ptr [[TMP28]], align 8
+; CHECK-NEXT: [[TMP49:%.*]] = fmul <2 x double> [[TMP47]], <double 1.000000e+00, double 0.000000e+00>
+; CHECK-NEXT: [[TMP50:%.*]] = fsub <2 x double> [[TMP49]], [[TMP38]]
+; CHECK-NEXT: store <2 x double> [[TMP50]], ptr [[TMP35]], align 8
; CHECK-NEXT: [[TMP39:%.*]] = fmul double [[TMP7]], 0.000000e+00
; CHECK-NEXT: [[TMP40:%.*]] = fadd double [[TMP39]], [[TMP1]]
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[TMP15]], i64 0
; CHECK-NEXT: [[TMP41:%.*]] = fadd double [[TMP40]], [[TMP10]]
; CHECK-NEXT: [[TMP42:%.*]] = fadd double [[TMP41]], 0.000000e+00
; CHECK-NEXT: [[TMP43:%.*]] = fadd double [[TMP14]], 0.000000e+00
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/trimmed-subtree-schedule-operands.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/trimmed-subtree-schedule-operands.ll
index 239672cfeda947..b529394ea0917a 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/trimmed-subtree-schedule-operands.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/trimmed-subtree-schedule-operands.ll
@@ -19,31 +19,30 @@ define void @test(ptr %srcGrid, ptr %dstGrid, i64 %iv) {
; CHECK-NEXT: [[ARRAYIDX148:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 64
; CHECK-NEXT: [[ARRAYIDX184:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 136
; CHECK-NEXT: [[ARRAYIDX188:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 144
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[ARRAYIDX184]], align 8
-; CHECK-NEXT: [[TMP4:%.*]] = load double, ptr [[ARRAYIDX188]], align 8
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> <double poison, double 0.000000e+00>, <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> <double poison, double 0.000000e+00>, <2 x double> [[TMP1]], <2 x i32> <i32 3, i32 1>
-; CHECK-NEXT: [[TMP5:%.*]] = fadd fast <2 x double> [[TMP3]], [[TMP9]]
-; CHECK-NEXT: [[TMP6:%.*]] = fsub fast <2 x double> zeroinitializer, [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = fadd fast double 0.000000e+00, [[TMP4]]
-; CHECK-NEXT: [[MOMENTUM_Z:%.*]] = fsub fast double 0.000000e+00, [[TMP7]]
; CHECK-NEXT: [[ARRAYIDX675:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[DSTGRID]], i64 [[IV]]
; CHECK-NEXT: [[ARRAYIDX787:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX675]], i64 32216
; CHECK-NEXT: [[TMP8:%.*]] = load double, ptr [[ARRAYIDX148]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[ARRAYIDX144]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = load double, ptr [[ARRAYIDX184]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = load double, ptr [[ARRAYIDX188]], align 8
; CHECK-NEXT: [[RHO:%.*]] = fadd fast double 1.000000e+00, [[TMP8]]
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x double> <double poison, double 1.000000e+00>, double [[RHO]], i64 0
-; CHECK-NEXT: [[TMP11:%.*]] = fdiv fast <2 x double> [[TMP6]], [[TMP13]]
+; CHECK-NEXT: [[TMP5:%.*]] = fadd fast double [[TMP3]], [[TMP4]]
+; CHECK-NEXT: [[MOMENTUM_Z:%.*]] = fsub fast double 0.000000e+00, [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = fadd fast double 0.000000e+00, 0.000000e+00
+; CHECK-NEXT: [[SUB266:%.*]] = fsub fast double 0.000000e+00, [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = fadd fast double 0.000000e+00, [[TMP4]]
+; CHECK-NEXT: [[MOMENTUM_Z1:%.*]] = fsub fast double 0.000000e+00, [[TMP7]]
; CHECK-NEXT: [[DIV307:%.*]] = fdiv fast double [[MOMENTUM_Z]], [[RHO]]
-; CHECK-NEXT: [[UZ_0:%.*]] = select nsz i1 true, double [[DIV307]], double 0.000000e+00
-; CHECK-NEXT: [[TMP12:%.*]] = select <2 x i1> zeroinitializer, <2 x double> [[TMP11]], <2 x double> zeroinitializer
+; CHECK-NEXT: [[DIV306:%.*]] = fdiv fast double [[SUB266]], 1.000000e+00
+; CHECK-NEXT: [[DIV308:%.*]] = fdiv fast double [[MOMENTUM_Z1]], [[RHO]]
+; CHECK-NEXT: [[UZ_0:%.*]] = select nsz i1 true, double [[DIV308]], double 0.000000e+00
+; CHECK-NEXT: [[TMP32:%.*]] = select nsz i1 false, double [[DIV306]], double 0.000000e+00
+; CHECK-NEXT: [[TMP31:%.*]] = select nsz i1 false, double [[DIV307]], double 0.000000e+00
; CHECK-NEXT: [[MUL317:%.*]] = fmul fast double [[UZ_0]], [[UZ_0]]
; CHECK-NEXT: [[ADD318:%.*]] = fadd fast double 0.000000e+00, [[MUL317]]
; CHECK-NEXT: [[MUL319:%.*]] = fmul fast double [[ADD318]], 1.000000e+00
; CHECK-NEXT: [[MUL348:%.*]] = fmul fast double [[RHO]], 1.000000e+00
-; CHECK-NEXT: [[UX_0:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[TMP12]])
-; CHECK-NEXT: [[TMP31:%.*]] = extractelement <2 x double> [[TMP12]], i64 0
-; CHECK-NEXT: [[TMP32:%.*]] = extractelement <2 x double> [[TMP12]], i64 1
+; CHECK-NEXT: [[UX_0:%.*]] = fadd fast double [[TMP31]], [[TMP32]]
; CHECK-NEXT: [[ADD359:%.*]] = fsub fast double [[TMP32]], [[TMP31]]
; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x double> poison, double [[UX_0]], i64 0
; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x double> [[TMP10]], double [[ADD359]], i64 1
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/gathered-load-strided-picks.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/gathered-load-strided-picks.ll
index a42ea4f2e1e9fa..7681511134e032 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/gathered-load-strided-picks.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/gathered-load-strided-picks.ll
@@ -10,7 +10,7 @@
; YAML-NEXT: Function: two_accumulators_f32
; YAML-NEXT: Args:
; YAML-NEXT: - String: 'SLP vectorized with cost '
-; YAML-NEXT: - Cost: '-16'
+; YAML-NEXT: - Cost: '-8'
; YAML-NEXT: - String: ' and with tree size '
; YAML-NEXT: - TreeSize: '17'
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/gathered-lane-of-vectorized-entry-zve32.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/gathered-lane-of-vectorized-entry-zve32.ll
new file mode 100644
index 00000000000000..7b2bab2ff2f001
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/gathered-lane-of-vectorized-entry-zve32.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -mtriple=riscv64-unknown-linux-gnu -mattr=+zve32f,+zvl128b < %s | FileCheck %s
+
+; With ELEN 32 the i64 shuffle has no valid cost, so the insert price stays.
+
+define void @mul_lane_and_arg(ptr %p, ptr %q, i64 %c) {
+; CHECK-LABEL: define void @mul_lane_and_arg(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i64 [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = xor <2 x i64> [[TMP1]], splat (i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i64> poison, i64 [[C]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP2]], <2 x i32> <i32 3, i32 1>
+; CHECK-NEXT: [[TMP5:%.*]] = mul <2 x i64> [[TMP2]], [[TMP4]]
+; CHECK-NEXT: store <2 x i64> [[TMP5]], ptr [[Q]], align 8
+; CHECK-NEXT: ret void
+;
+ %p1 = getelementptr inbounds i64, ptr %p, i64 1
+ %a0 = load i64, ptr %p, align 8
+ %a1 = load i64, ptr %p1, align 8
+ %x0 = xor i64 %a0, 1
+ %x1 = xor i64 %a1, 1
+ %z0 = zext i64 %x0 to i128
+ %z1 = zext i64 %x1 to i128
+ %w0 = zext i64 %x1 to i128
+ %w1 = zext i64 %c to i128
+ %m0 = mul nuw i128 %z0, %w0
+ %m1 = mul nuw i128 %z1, %w1
+ %t0 = trunc i128 %m0 to i64
+ %t1 = trunc i128 %m1 to i64
+ %q1 = getelementptr inbounds i64, ptr %q, i64 1
+ store i64 %t0, ptr %q, align 8
+ store i64 %t1, ptr %q1, align 8
+ ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gathered-shuffle-resized.ll b/llvm/test/Transforms/SLPVectorizer/X86/gathered-shuffle-resized.ll
index 396fd161aa8885..1204ae0f31254d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/gathered-shuffle-resized.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/gathered-shuffle-resized.ll
@@ -15,12 +15,13 @@ define ptr @test(ptr %0, ptr %args_gep) {
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[ARG26]], i64 17
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[ARG1]], i64 8
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[ARG1]], i64 12
-; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !noalias [[META0:![0-9]+]]
-; CHECK-NEXT: [[TMP9:%.*]] = load <2 x i32>, ptr [[TMP6]], align 8, !noalias [[META0]]
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <16 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <16 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <16 x i32> [[TMP10]], <16 x i32> [[TMP11]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 16, i32 8, i32 9, i32 10, i32 11, i32 12, i32 22, i32 14, i32 15>
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <16 x i32> [[TMP12]], <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 13, i32 13, i32 13>
+; CHECK-NEXT: [[TMP9:%.*]] = load <2 x i32>, ptr [[TMP6]], align 8, !noalias [[META0:![0-9]+]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !noalias [[META0]]
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <7 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <9 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1>
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <7 x i32> [[TMP10]], <7 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <9 x i32> [[TMP11]], <9 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <16 x i32> [[TMP12]], <16 x i32> [[TMP18]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24>
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq <16 x i32> [[TMP13]], zeroinitializer
; CHECK-NEXT: [[TMP15:%.*]] = zext <16 x i1> [[TMP14]] to <16 x i8>
; CHECK-NEXT: store <16 x i8> [[TMP15]], ptr [[TMP5]], align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/absorbed-lane-shuffled-operand.ll b/llvm/test/Transforms/SLPVectorizer/absorbed-lane-shuffled-operand.ll
index afd7133d65d145..88ba5b58bf8779 100644
--- a/llvm/test/Transforms/SLPVectorizer/absorbed-lane-shuffled-operand.ll
+++ b/llvm/test/Transforms/SLPVectorizer/absorbed-lane-shuffled-operand.ll
@@ -27,8 +27,7 @@ define i32 @swap_bytes(ptr %params, ptr %out) {
; CHECK-NEXT: [[TMP2:%.*]] = sub <2 x i32> [[TMP0]], [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
; CHECK-NEXT: [[TMP4:%.*]] = and <8 x i32> [[TMP3]], <i32 -1, i32 1, i32 253, i32 3, i32 251, i32 5, i32 249, i32 7>
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x i32> <i32 -1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 1, i32 2, i32 1, i32 2, i32 1>
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> <i32 -1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, <8 x i32> <i32 8, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0>
; CHECK-NEXT: [[TMP8:%.*]] = and <8 x i32> <i32 0, i32 254, i32 2, i32 252, i32 4, i32 250, i32 6, i32 248>, [[TMP6]]
; CHECK-NEXT: [[TMP9:%.*]] = or disjoint <8 x i32> [[TMP4]], [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = trunc <8 x i32> [[TMP9]] to <8 x i8>
diff --git a/llvm/test/Transforms/SLPVectorizer/extract-many-users-buildvector.ll b/llvm/test/Transforms/SLPVectorizer/extract-many-users-buildvector.ll
index 583ada6022f3ee..d8cd996a0e6044 100644
--- a/llvm/test/Transforms/SLPVectorizer/extract-many-users-buildvector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/extract-many-users-buildvector.ll
@@ -33,28 +33,27 @@ define i1 @test(float %0, double %1) {
; AARCH64-NEXT: [[TMP3:%.*]] = insertelement <2 x float> <float 0.000000e+00, float poison>, float [[TMP0]], i64 1
; AARCH64-NEXT: [[TMP4:%.*]] = fpext <2 x float> [[TMP3]] to <2 x double>
; AARCH64-NEXT: [[TMP5:%.*]] = fmul double 0.000000e+00, 0.000000e+00
-; AARCH64-NEXT: [[TMP6:%.*]] = insertelement <8 x double> <double poison, double poison, double 0.000000e+00, double poison, double poison, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>, double [[TMP1]], i64 4
-; AARCH64-NEXT: [[TMP7:%.*]] = insertelement <8 x double> [[TMP6]], double 0.000000e+00, i64 0
-; AARCH64-NEXT: [[TMP8:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> poison, <8 x i32> <i32 0, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AARCH64-NEXT: [[TMP9:%.*]] = shufflevector <8 x double> [[TMP7]], <8 x double> [[TMP8]], <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 5, i32 6, i32 7>
-; AARCH64-NEXT: [[TMP10:%.*]] = fmul <8 x double> zeroinitializer, [[TMP9]]
-; AARCH64-NEXT: [[TMP11:%.*]] = shufflevector <8 x double> [[TMP9]], <8 x double> <double 0.000000e+00, double poison, double poison, double poison, double 0.000000e+00, double 0.000000e+00, double 0.000000e+00, double 0.000000e+00>, <8 x i32> <i32 8, i32 4, i32 poison, i32 poison, i32 12, i32 13, i32 14, i32 15>
-; AARCH64-NEXT: [[TMP12:%.*]] = insertelement <8 x double> poison, double [[TMP5]], i64 0
-; AARCH64-NEXT: [[TMP13:%.*]] = shufflevector <8 x double> [[TMP12]], <8 x double> poison, <8 x i32> <i32 poison, i32 poison, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison>
-; AARCH64-NEXT: [[TMP14:%.*]] = shufflevector <8 x double> [[TMP11]], <8 x double> [[TMP13]], <8 x i32> <i32 0, i32 1, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; AARCH64-NEXT: [[TMP15:%.*]] = shufflevector <8 x double> [[TMP14]], <8 x double> <double poison, double poison, double poison, double 0.000000e+00, double 1.000000e+00, double poison, double 0.000000e+00, double 0.000000e+00>, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 11, i32 12, i32 2, i32 14, i32 15>
-; AARCH64-NEXT: [[TMP16:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AARCH64-NEXT: [[TMP17:%.*]] = shufflevector <8 x double> [[TMP16]], <8 x double> poison, <8 x i32> <i32 poison, i32 poison, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AARCH64-NEXT: [[TMP18:%.*]] = shufflevector <8 x double> [[TMP15]], <8 x double> [[TMP17]], <8 x i32> <i32 poison, i32 poison, i32 10, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AARCH64-NEXT: [[TMP19:%.*]] = shufflevector <8 x double> [[TMP18]], <8 x double> <double 0.000000e+00, double 0.000000e+00, double poison, double poison, double poison, double poison, double poison, double poison>, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AARCH64-NEXT: [[TMP20:%.*]] = fmul <8 x double> [[TMP14]], [[TMP19]]
-; AARCH64-NEXT: [[TMP21:%.*]] = fsub <8 x double> [[TMP20]], [[TMP10]]
-; AARCH64-NEXT: [[TMP22:%.*]] = fptrunc <8 x double> [[TMP21]] to <8 x float>
-; AARCH64-NEXT: [[TMP23:%.*]] = fmul <8 x float> [[TMP22]], zeroinitializer
-; AARCH64-NEXT: [[TMP24:%.*]] = fcmp oeq <8 x float> [[TMP23]], zeroinitializer
-; AARCH64-NEXT: [[TMP25:%.*]] = freeze <8 x i1> [[TMP24]]
-; AARCH64-NEXT: [[TMP26:%.*]] = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> [[TMP25]])
-; AARCH64-NEXT: ret i1 [[TMP26]]
+; AARCH64-NEXT: [[TMP6:%.*]] = insertelement <8 x double> <double 0.000000e+00, double poison, double 0.000000e+00, double poison, double poison, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>, double [[TMP1]], i64 4
+; AARCH64-NEXT: [[TMP7:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> poison, <8 x i32> <i32 0, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AARCH64-NEXT: [[TMP8:%.*]] = shufflevector <8 x double> [[TMP6]], <8 x double> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 5, i32 6, i32 7>
+; AARCH64-NEXT: [[TMP9:%.*]] = fmul <8 x double> zeroinitializer, [[TMP8]]
+; AARCH64-NEXT: [[TMP10:%.*]] = shufflevector <8 x double> [[TMP8]], <8 x double> <double 0.000000e+00, double poison, double poison, double poison, double 0.000000e+00, double 0.000000e+00, double 0.000000e+00, double 0.000000e+00>, <8 x i32> <i32 8, i32 4, i32 poison, i32 poison, i32 12, i32 13, i32 14, i32 15>
+; AARCH64-NEXT: [[TMP11:%.*]] = insertelement <8 x double> poison, double [[TMP5]], i64 0
+; AARCH64-NEXT: [[TMP12:%.*]] = shufflevector <8 x double> [[TMP11]], <8 x double> poison, <8 x i32> <i32 poison, i32 poison, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison>
+; AARCH64-NEXT: [[TMP13:%.*]] = shufflevector <8 x double> [[TMP10]], <8 x double> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; AARCH64-NEXT: [[TMP14:%.*]] = shufflevector <8 x double> [[TMP13]], <8 x double> <double poison, double poison, double poison, double 0.000000e+00, double 1.000000e+00, double poison, double 0.000000e+00, double 0.000000e+00>, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 11, i32 12, i32 2, i32 14, i32 15>
+; AARCH64-NEXT: [[TMP15:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AARCH64-NEXT: [[TMP16:%.*]] = shufflevector <8 x double> [[TMP15]], <8 x double> poison, <8 x i32> <i32 poison, i32 poison, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AARCH64-NEXT: [[TMP17:%.*]] = shufflevector <8 x double> [[TMP14]], <8 x double> [[TMP16]], <8 x i32> <i32 poison, i32 poison, i32 10, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AARCH64-NEXT: [[TMP18:%.*]] = shufflevector <8 x double> [[TMP17]], <8 x double> <double 0.000000e+00, double 0.000000e+00, double poison, double poison, double poison, double poison, double poison, double poison>, <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AARCH64-NEXT: [[TMP19:%.*]] = fmul <8 x double> [[TMP13]], [[TMP18]]
+; AARCH64-NEXT: [[TMP20:%.*]] = fsub <8 x double> [[TMP19]], [[TMP9]]
+; AARCH64-NEXT: [[TMP21:%.*]] = fptrunc <8 x double> [[TMP20]] to <8 x float>
+; AARCH64-NEXT: [[TMP22:%.*]] = fmul <8 x float> [[TMP21]], zeroinitializer
+; AARCH64-NEXT: [[TMP23:%.*]] = fcmp oeq <8 x float> [[TMP22]], zeroinitializer
+; AARCH64-NEXT: [[TMP24:%.*]] = freeze <8 x i1> [[TMP23]]
+; AARCH64-NEXT: [[TMP25:%.*]] = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> [[TMP24]])
+; AARCH64-NEXT: ret i1 [[TMP25]]
;
%3 = fpext float %0 to double
%4 = fpext float 0.000000e+00 to double
diff --git a/llvm/test/Transforms/SLPVectorizer/gathered-consecutive-loads-different-types.ll b/llvm/test/Transforms/SLPVectorizer/gathered-consecutive-loads-different-types.ll
index 2c784b3b7ca58a..d4cfb020e2e0d8 100644
--- a/llvm/test/Transforms/SLPVectorizer/gathered-consecutive-loads-different-types.ll
+++ b/llvm/test/Transforms/SLPVectorizer/gathered-consecutive-loads-different-types.ll
@@ -12,7 +12,7 @@ define i32 @test(i8 %0) {
; CHECK-NEXT: [[TMP12:%.*]] = load i8, ptr addrspace(21) null, align 2
; CHECK-NEXT: [[TMP13:%.*]] = load volatile i8, ptr null, align 2
; CHECK-NEXT: [[TMP5:%.*]] = load <2 x i8>, ptr addrspace(21) getelementptr inbounds (i8, ptr addrspace(21) null, i64 7), align 1
-; CHECK-NEXT: [[TMP32:%.*]] = shufflevector <2 x i8> <i8 0, i8 poison>, <2 x i8> [[TMP5]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: [[TMP32:%.*]] = shufflevector <2 x i8> [[TMP5]], <2 x i8> <i8 0, i8 poison>, <2 x i32> <i32 2, i32 0>
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <2 x i8> zeroinitializer, [[TMP32]]
; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i8> [[TMP5]], <2 x i8> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP33:%.*]] = shufflevector <8 x i8> <i8 0, i8 0, i8 poison, i8 0, i8 0, i8 poison, i8 0, i8 0>, <8 x i8> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 9, i32 3, i32 4, i32 poison, i32 6, i32 7>
@@ -22,9 +22,9 @@ define i32 @test(i8 %0) {
; CHECK-NEXT: [[TMP10:%.*]] = trunc <4 x i48> [[TMP9]] to <4 x i8>
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq <4 x i8> zeroinitializer, [[TMP10]]
; CHECK-NEXT: [[TMP14:%.*]] = load <2 x i8>, ptr addrspace(21) getelementptr inbounds (i8, ptr addrspace(21) null, i64 8), align 8
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <8 x i8> <i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 0, i8 0, i8 0>, i8 [[TMP12]], i64 1
-; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <2 x i8> [[TMP14]], <2 x i8> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <8 x i8> [[TMP15]], <8 x i8> [[TMP16]], <8 x i32> <i32 0, i32 1, i32 2, i32 8, i32 9, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <2 x i8> [[TMP14]], <2 x i8> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <8 x i8> [[TMP16]], <8 x i8> <i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 0, i8 0, i8 0>, <8 x i32> <i32 8, i32 poison, i32 10, i32 3, i32 4, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <8 x i8> [[TMP18]], i8 [[TMP12]], i64 1
; CHECK-NEXT: [[TMP34:%.*]] = insertelement <8 x i8> <i8 0, i8 poison, i8 0, i8 poison, i8 0, i8 0, i8 0, i8 0>, i8 [[TMP0]], i64 3
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <8 x i8> [[TMP34]], i8 [[TMP13]], i64 1
; CHECK-NEXT: [[TMP20:%.*]] = icmp eq <8 x i8> [[TMP17]], [[TMP19]]
diff --git a/llvm/test/Transforms/SLPVectorizer/gathered-swapped-vectorized-lanes.ll b/llvm/test/Transforms/SLPVectorizer/gathered-swapped-vectorized-lanes.ll
index 0be3c7cfbc6e88..323ce03a3fe2ef 100644
--- a/llvm/test/Transforms/SLPVectorizer/gathered-swapped-vectorized-lanes.ll
+++ b/llvm/test/Transforms/SLPVectorizer/gathered-swapped-vectorized-lanes.ll
@@ -6,14 +6,15 @@
define void @swapped_operand_f32(ptr %a, ptr %out) {
; CHECK-LABEL: define void @swapped_operand_f32(
; CHECK-SAME: ptr [[A:%.*]], ptr [[OUT:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[A]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = fmul <2 x float> [[TMP1]], <float 2.000000e+00, float 3.000000e+00>
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT: [[TMP4:%.*]] = fsub <2 x float> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr float, ptr [[A]], i64 1
+; CHECK-NEXT: [[A0:%.*]] = load float, ptr [[A]], align 4
+; CHECK-NEXT: [[A1:%.*]] = load float, ptr [[P1]], align 4
+; CHECK-NEXT: [[T0:%.*]] = fmul float [[A0]], 2.000000e+00
+; CHECK-NEXT: [[T1:%.*]] = fmul float [[A1]], 3.000000e+00
+; CHECK-NEXT: [[TMP5:%.*]] = fsub float [[T0]], [[T1]]
+; CHECK-NEXT: [[TMP6:%.*]] = fsub float [[T1]], [[T0]]
; CHECK-NEXT: [[O1:%.*]] = getelementptr float, ptr [[OUT]], i64 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x float> [[TMP4]], i64 0
; CHECK-NEXT: store float [[TMP5]], ptr [[OUT]], align 4
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x float> [[TMP4]], i64 1
; CHECK-NEXT: store float [[TMP6]], ptr [[O1]], align 4
; CHECK-NEXT: ret void
;
@@ -33,14 +34,15 @@ define void @swapped_operand_f32(ptr %a, ptr %out) {
define void @swapped_operand_i32(ptr %a, ptr %out) {
; CHECK-LABEL: define void @swapped_operand_i32(
; CHECK-SAME: ptr [[A:%.*]], ptr [[OUT:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[A]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = mul <2 x i32> [[TMP1]], <i32 2, i32 3>
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT: [[TMP4:%.*]] = sub <2 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[P1:%.*]] = getelementptr i32, ptr [[A]], i64 1
+; CHECK-NEXT: [[A0:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT: [[A1:%.*]] = load i32, ptr [[P1]], align 4
+; CHECK-NEXT: [[T0:%.*]] = mul i32 [[A0]], 2
+; CHECK-NEXT: [[T1:%.*]] = mul i32 [[A1]], 3
+; CHECK-NEXT: [[TMP5:%.*]] = sub i32 [[T0]], [[T1]]
+; CHECK-NEXT: [[TMP6:%.*]] = sub i32 [[T1]], [[T0]]
; CHECK-NEXT: [[O1:%.*]] = getelementptr i32, ptr [[OUT]], i64 1
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i32> [[TMP4]], i64 0
; CHECK-NEXT: store i32 [[TMP5]], ptr [[OUT]], align 4
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[TMP4]], i64 1
; CHECK-NEXT: store i32 [[TMP6]], ptr [[O1]], align 4
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/resized-alt-shuffle-after-minbw.ll b/llvm/test/Transforms/SLPVectorizer/resized-alt-shuffle-after-minbw.ll
index bf643b5dc1ec85..ecbb62b6fbdea4 100644
--- a/llvm/test/Transforms/SLPVectorizer/resized-alt-shuffle-after-minbw.ll
+++ b/llvm/test/Transforms/SLPVectorizer/resized-alt-shuffle-after-minbw.ll
@@ -10,12 +10,12 @@ define void @func(i32 %0) {
; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <32 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[TMP9:%.*]] = sext i32 [[TMP6]] to i64
; CHECK-NEXT: [[TMP10:%.*]] = or i64 [[TMP9]], 0
-; CHECK-NEXT: [[TMP11:%.*]] = trunc i64 [[TMP9]] to i32
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <32 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, i32 [[TMP11]], i64 30
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <32 x i32> [[TMP12]], <32 x i32> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 30, i32 30>
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <32 x i32> [[TMP13]], <32 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 poison, i32 poison, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 30, i32 31>
-; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <32 x i32> [[TMP14]], <32 x i32> <i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 poison, i32 poison, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 poison, i32 poison, i32 30, i32 31>
-; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <32 x i32> [[TMP15]], <32 x i32> <i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 32, i32 33, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 poison, i32 poison, i32 30, i32 31>
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <32 x i32> [[TMP13]], <32 x i32> <i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, <32 x i32> <i32 poison, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 1>
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <32 x i32> [[TMP14]], i32 0, i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <32 x i32> [[TMP15]], <32 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <32 x i32> [[TMP11]], <32 x i32> <i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 32, i32 33, i32 34, i32 35, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <32 x i32> [[TMP12]], <32 x i32> <i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 32, i32 33, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <32 x i32> [[TMP16]], <32 x i32> <i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 32, i32 33, i32 30, i32 31>
; CHECK-NEXT: [[TMP18:%.*]] = or <32 x i32> [[TMP8]], [[TMP17]]
; CHECK-NEXT: [[TMP20:%.*]] = icmp slt <32 x i32> [[TMP18]], zeroinitializer
More information about the llvm-branch-commits
mailing list