[llvm] [SLP]Vectorize single-user instructions as the last-attempt seeds (PR #212579)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 4 08:51:04 PDT 2026
https://github.com/alexey-bataev updated https://github.com/llvm/llvm-project/pull/212579
>From 8673d9b10ce791aff69fe2ccb2e872b04ddebe20 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Tue, 28 Jul 2026 11:44:20 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../llvm/Transforms/Vectorize/SLPVectorizer.h | 16 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 386 ++++++++++++++----
.../Vectorize/SLPVectorizer/SLPUtils.cpp | 54 +++
.../Vectorize/SLPVectorizer/SLPUtils.h | 10 +
llvm/test/Transforms/PhaseOrdering/X86/avg.ll | 239 ++++++-----
.../AArch64/extractelements-to-shuffle.ll | 22 +-
.../AArch64/unsigned-after-sext-node.ll | 8 +-
.../vectorize-free-extracts-inserts.ll | 16 +-
.../Transforms/SLPVectorizer/RISCV/revec.ll | 19 +-
.../RISCV/runtime-strided-stores.ll | 6 +-
.../X86/alternate-calls-inseltpoison.ll | 26 +-
.../SLPVectorizer/X86/alternate-calls.ll | 28 +-
.../X86/alternate-int-inseltpoison.ll | 18 +-
.../SLPVectorizer/X86/alternate-int.ll | 18 +-
.../X86/arith-fp-inseltpoison.ll | 43 +-
.../Transforms/SLPVectorizer/X86/arith-fp.ll | 43 +-
.../Transforms/SLPVectorizer/X86/broadcast.ll | 9 +-
.../Transforms/SLPVectorizer/X86/c-ray.ll | 22 +-
.../X86/copyable-operands-reordering.ll | 12 +-
.../X86/crash_getpointersdiff-nullopt.ll | 14 +-
.../SLPVectorizer/X86/debug-info-salvage.ll | 15 +-
.../X86/deleted-inst-reduction-attempt.ll | 11 +-
.../X86/extractelement-multi-register-use.ll | 20 +-
.../SLPVectorizer/X86/horizontal-list.ll | 10 +-
.../Transforms/SLPVectorizer/X86/intrinsic.ll | 99 ++---
.../Transforms/SLPVectorizer/X86/lookahead.ll | 11 +-
.../X86/non-vectorizable-inst-operand.ll | 9 +-
.../X86/parent-node-split-non-schedulable.ll | 10 +-
.../X86/phi-comparator-fix-vec-ops-compare.ll | 14 +-
.../SLPVectorizer/X86/pr42022-inseltpoison.ll | 17 +-
.../Transforms/SLPVectorizer/X86/pr42022.ll | 17 +-
.../Transforms/SLPVectorizer/X86/pr52275.ll | 46 +--
.../X86/reorder_diamond_match.ll | 50 +--
.../X86/reused-mask-with-poison-index.ll | 71 ++--
.../SLPVectorizer/X86/scalarize-ctlz.ll | 20 +-
.../SLPVectorizer/alternate-non-profitable.ll | 18 +-
...nsert-element-build-vector-inseltpoison.ll | 78 ++--
.../insert-element-build-vector.ll | 79 ++--
...minbitwidth-multiuse-with-insertelement.ll | 9 +-
39 files changed, 944 insertions(+), 669 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index 538d62626b37b..468cec660343c 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -94,10 +94,12 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
/// Try to vectorize a list of operands.
/// \param MaxVFOnly Vectorize only using maximal allowed register size.
+ /// \param StandaloneSeeds \p VL are the standalone seeds: the vector factor
+ /// is limited by a single register and the windows, overlapping with the
+ /// rejected ones, are not retried.
/// \returns true if a value was vectorized.
bool tryToVectorizeList(ArrayRef<Value *> VL, slpvectorizer::BoUpSLP &R,
- bool MaxVFOnly = false,
- bool LimitToRegisterVF = false);
+ bool MaxVFOnly = false, bool StandaloneSeeds = false);
/// Try to vectorize a chain that may start at the operands of \p I.
bool tryToVectorize(Instruction *I, slpvectorizer::BoUpSLP &R,
@@ -112,6 +114,16 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
/// Vectorize the store instructions collected in Stores.
bool vectorizeStoreChains(slpvectorizer::BoUpSLP &R);
+ /// Try to vectorize the standalone seeds \p Seeds in the groups of the
+ /// compatible instructions, \p IsLessGroup orders the groups.
+ bool vectorizeSeeds(SmallVectorImpl<Value *> &Seeds,
+ function_ref<bool(Value *, Value *)> IsLessGroup,
+ slpvectorizer::BoUpSLP &R);
+
+ /// Try to vectorize the instructions with the single user in \p BB as the
+ /// standalone seeds.
+ bool vectorizeOnceUsedSeeds(BasicBlock *BB, slpvectorizer::BoUpSLP &R);
+
/// Vectorize the index computations of the getelementptr instructions
/// collected in GEPs.
bool vectorizeGEPIndices(BasicBlock *BB, slpvectorizer::BoUpSLP &R);
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 1a26b6ae4e52d..a9bf2c9783e2e 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -264,6 +264,11 @@ static cl::opt<bool> VectorizePoorThroughput(
cl::desc("Use poor-throughput instructions (e.g. fdiv, frem, fsqrt) as "
"standalone vectorization seeds."));
+static cl::opt<bool> VectorizeOnceUsed(
+ "slp-vectorize-once-used", cl::init(true), cl::Hidden,
+ cl::desc("Use instructions with the single user as standalone "
+ "vectorization seeds."));
+
/// True when \p slp-vectorize-non-power-of-2 is enabled and \p NumElts is a
/// supported non-power-of-2 width: \p NumElts + 1 must be a power of two
/// (e.g. 3 or 7 lanes, i.e. almost a full power-of-2 register).
@@ -1567,6 +1572,19 @@ class slpvectorizer::BoUpSLP {
return MinVecRegSize;
}
+ /// \returns the number of parts, the type \p VecTy is split at the codegen
+ /// phase. The type legalization queries are repeated for the very same types
+ /// during the analysis, so the results are cached for the function.
+ unsigned getNumberOfParts(
+ Type *VecTy, Type *ScalarTy,
+ unsigned Limit = std::numeric_limits<unsigned>::max()) const {
+ auto [It, Inserted] =
+ NumberOfPartsCache.try_emplace(std::make_tuple(VecTy, ScalarTy, Limit));
+ if (Inserted)
+ It->second = ::getNumberOfParts(*TTI, VecTy, ScalarTy, Limit);
+ return It->second;
+ }
+
unsigned getMinVF(unsigned Sz) const {
return std::max(2U, getMinVecRegSize() / Sz);
}
@@ -2944,6 +2962,9 @@ class slpvectorizer::BoUpSLP {
/// Checks if the instruction is marked for deletion.
bool isDeleted(Instruction *I) const { return DeletedInstructions.count(I); }
+ /// Checks if the value is used only by the assume-like intrinsics.
+ bool isEphemeralValue(const Value *V) const { return EphValues.contains(V); }
+
/// Removes an instruction from its block and eventually deletes it.
/// It's like Instruction::eraseFromParent() except that the actual deletion
/// is delayed until BoUpSLP is destructed.
@@ -3058,10 +3079,23 @@ class slpvectorizer::BoUpSLP {
void analyzedReductionVals(ArrayRef<Value *> VL) {
AnalyzedReductionVals.insert(hash_value(VL));
}
+ /// Checks if the value was already a part of the analyzed vector node.
+ bool isAnalyzedScalar(const Value *V) const {
+ return AnalyzedScalars.contains(V);
+ }
+ /// Checks if the given bundle was already rejected as non-vectorizable.
+ bool isAnalyzedBundle(ArrayRef<Value *> VL) const {
+ return AnalyzedBundles.contains(hash_value(VL));
+ }
+ /// Registers the bundle as rejected for the vectorization.
+ void analyzedBundle(ArrayRef<Value *> VL) {
+ AnalyzedBundles.insert(hash_value(VL));
+ }
/// Clear the list of the analyzed reduction root instructions.
void clearReductionData() {
AnalyzedReductionsRoots.clear();
AnalyzedReductionVals.clear();
+ AnalyzedBundles.clear();
AnalyzedMinBWVals.clear();
}
/// Checks if the given value is gathered in one of the nodes.
@@ -4327,6 +4361,16 @@ class slpvectorizer::BoUpSLP {
/// Set of hashes for the list of reduction values already being analyzed.
DenseSet<size_t> AnalyzedReductionVals;
+ /// Set of hashes for the bundles, rejected as non-vectorizable.
+ SmallDenseSet<size_t, 8> AnalyzedBundles;
+
+ /// Set of the values, which were a part of the analyzed vector nodes.
+ SmallPtrSet<const Value *, 32> AnalyzedScalars;
+
+ /// Cache of the number of parts for the types and the parts limit.
+ mutable SmallDenseMap<std::tuple<Type *, Type *, unsigned>, unsigned>
+ NumberOfPartsCache;
+
/// Values, already been analyzed for mininmal bitwidth and found to be
/// non-profitable.
DenseSet<Value *> AnalyzedMinBWVals;
@@ -6069,7 +6113,7 @@ BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE,
if (!isValidElementType(ScalarTy))
return std::nullopt;
auto *VecTy = getWidenedType(ScalarTy, NumScalars);
- unsigned NumParts = ::getNumberOfParts(*TTI, VecTy, ScalarTy, NumScalars);
+ unsigned NumParts = getNumberOfParts(VecTy, ScalarTy, NumScalars);
SmallVector<int> ExtractMask;
SmallVector<int> Mask;
SmallVector<SmallVector<const TreeEntry *>> Entries;
@@ -10123,11 +10167,6 @@ struct SeedGroupKey {
Intrinsic::ID IntrID = Intrinsic::not_intrinsic;
StringRef CalleeName;
- bool operator==(const SeedGroupKey &O) const {
- return Opcode == O.Opcode && IntrID == O.IntrID &&
- CalleeName == O.CalleeName;
- }
- bool operator!=(const SeedGroupKey &O) const { return !(*this == O); }
bool less(const SeedGroupKey &O) const {
if (Opcode != O.Opcode)
return Opcode < O.Opcode;
@@ -10150,6 +10189,15 @@ static SeedGroupKey getSeedGroupKey(const Instruction *I,
}
} // namespace
+/// Returns true if the role of \p I is already decided by its user: a deleted
+/// user was folded into some other vector by an earlier attempt.
+static bool hasResolvedUser(Instruction *I, const BoUpSLP &R) {
+ return any_of(I->users(), [&](User *U) {
+ auto *UI = dyn_cast<Instruction>(U);
+ return UI && (R.isDeleted(UI) || R.isVectorized(UI));
+ });
+}
+
/// Returns true if \p I is an expensive scalar op whose vector form is cheaper
/// per lane (e.g. fdiv, frem, fsqrt).
static bool isPoorThroughputOp(Instruction *I, const TargetTransformInfo &TTI,
@@ -16309,7 +16357,7 @@ BoUpSLP::getVectorSpillReloadCost(const TreeEntry *E, Type *ScalarTy,
if (!CountedOpEntries.insert(OpTE).second)
continue;
auto [ScalarTy, OpVecTy] = GetEntryVecTy(OpTE);
- const unsigned Parts = ::getNumberOfParts(*TTI, OpVecTy, ScalarTy);
+ const unsigned Parts = getNumberOfParts(OpVecTy, ScalarTy);
if (Parts == 0)
continue;
const unsigned RC =
@@ -16323,7 +16371,7 @@ BoUpSLP::getVectorSpillReloadCost(const TreeEntry *E, Type *ScalarTy,
for (unsigned Idx : seq<unsigned>(E->getNumOperands())) {
const TreeEntry *OpTE = getOperandEntry(E, Idx);
auto [ScalarTy, OpVecTy] = GetEntryVecTy(OpTE);
- const unsigned Parts = ::getNumberOfParts(*TTI, OpVecTy, ScalarTy);
+ const unsigned Parts = getNumberOfParts(OpVecTy, ScalarTy);
if (Parts == 0)
continue;
const unsigned RC =
@@ -16351,7 +16399,7 @@ BoUpSLP::getVectorSpillReloadCost(const TreeEntry *E, Type *ScalarTy,
if (!CountedOpEntries.insert(OpTE).second)
continue;
auto *OpVecTy = getWidenedType(Op->getType(), Ops.size());
- const unsigned Parts = ::getNumberOfParts(*TTI, OpVecTy, Op->getType());
+ const unsigned Parts = getNumberOfParts(OpVecTy, Op->getType());
if (Parts == 0)
continue;
const unsigned RC =
@@ -16361,14 +16409,13 @@ BoUpSLP::getVectorSpillReloadCost(const TreeEntry *E, Type *ScalarTy,
}
if (E->getOpcode() != Instruction::Load) {
- const unsigned ResParts = ::getNumberOfParts(*TTI, VecTy, ScalarTy);
+ const unsigned ResParts = getNumberOfParts(VecTy, ScalarTy);
if (ResParts != 0) {
const unsigned RC = TTI->getRegisterClassForType(/*Vector=*/true, VecTy);
AddPartsToClass(RC, ResParts);
}
if (VecTy != FinalVecTy) {
- const unsigned FinalResParts =
- ::getNumberOfParts(*TTI, FinalVecTy, ScalarTy);
+ const unsigned FinalResParts = getNumberOfParts(FinalVecTy, ScalarTy);
if (FinalResParts != 0) {
const unsigned RC =
TTI->getRegisterClassForType(/*Vector=*/true, FinalVecTy);
@@ -16758,7 +16805,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
unsigned const NumScalars = VL.size();
unsigned NumOfParts =
- ::getNumberOfParts(*TTI, SrcVecTy, VL0->getOperand(1)->getType());
+ getNumberOfParts(SrcVecTy, VL0->getOperand(1)->getType());
SmallVector<int> InsertMask(NumElts, PoisonMaskElem);
unsigned OffsetBeg = *getElementIndex(VL.front());
@@ -17871,6 +17918,25 @@ bool BoUpSLP::isTreeTinyAndNotFullyVectorizable(bool ForReduction) const {
// a single guard so reduction trees skip them with one branch instead of one
// per check.
if (!ForReduction) {
+ // The single vectorized node, fed by the buildvectors, only repacks the
+ // scalars, the gathers and the extracts are paid for the one vector
+ // instruction. The vector call replaces the several expensive scalar ones,
+ // while the store merges the lanes instead of extracting them back.
+ if (TreeSize > 1 && ThresholdNonNegative &&
+ (!FrontHasState || (FrontOpcode != Instruction::Call &&
+ FrontOpcode != Instruction::Store)) &&
+ count_if(VectorizableTree,
+ [](const std::unique_ptr<TreeEntry> &TE) {
+ return TE->State == TreeEntry::Vectorize;
+ }) == 1 &&
+ all_of(VectorizableTree, [](const std::unique_ptr<TreeEntry> &TE) {
+ return TE->State == TreeEntry::Vectorize ||
+ (TE->isGather() && !isSplat(TE->Scalars) &&
+ !all_of(TE->Scalars,
+ IsaPred<ExtractElementInst, UndefValue, Constant>));
+ }))
+ return true;
+
// If the graph includes only PHI nodes and gathers, it is defnitely not
// profitable for the vectorization, we can skip it, if the cost threshold
// is default. The cost of vectorized PHI nodes is almost always 0 + the
@@ -18717,6 +18783,27 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals,
return InstructionCost::getInvalid();
}
+ // The tree is priced, so its minimal vector nodes are analyzed, including the
+ // ones, trimmed below as non-profitable. The wider nodes are not registered:
+ // their rejection does not prevent the members from forming the profitable
+ // narrower node. Same for the trees, rejected before the cost estimation, and
+ // for the buildvector trees, rejected by the insert/extract overhead rather
+ // than by the members themselves.
+ const TreeEntry &Root = *VectorizableTree.front();
+ LLVM_DEBUG(dbgs() << "SLP: Analyzed-scalar registration, root "
+ << (Root.hasState() ? Root.getOpcode() : unsigned(0))
+ << " size " << VectorizableTree.size() << ".\n");
+ if (!Root.hasState() ||
+ !isa<InsertElementInst, InsertValueInst>(Root.getMainOp())) {
+ for (const std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
+ if (TE->isGather() || TE->Scalars.size() > 2)
+ continue;
+ for (Value *V : TE->Scalars)
+ if (isa_and_present<Instruction>(V) && !TE->isCopyableElement(V))
+ AnalyzedScalars.insert(V);
+ }
+ }
+
SmallDenseMap<const TreeEntry *, InstructionCost> NodesCosts;
SmallPtrSet<Value *, 4> CheckedExtracts;
SmallSetVector<TreeEntry *, 4> GatheredLoadsNodes;
@@ -20823,7 +20910,7 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
*TTI, VL.front()->getType(), MinIdx);
auto *RegFloorTy = getWidenedType(VL.front()->getType(), RegFloor);
unsigned RegFloorParts =
- ::getNumberOfParts(*TTI, RegFloorTy, VL.front()->getType(), RegFloor);
+ getNumberOfParts(RegFloorTy, VL.front()->getType(), RegFloor);
if (RegFloorParts > 1)
Offset = RegFloor;
}
@@ -22192,8 +22279,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
SmallVector<SmallVector<const TreeEntry *>> Entries;
Type *OrigScalarTy = GatheredScalars.front()->getType();
auto *VecTy = getWidenedType(ScalarTy, GatheredScalars.size());
- unsigned NumParts =
- ::getNumberOfParts(*TTI, VecTy, ScalarTy, GatheredScalars.size());
+ unsigned NumParts = getNumberOfParts(VecTy, ScalarTy, GatheredScalars.size());
if (!all_of(GatheredScalars, IsaPred<UndefValue>)) {
// Check for gathered extracts.
bool Resized = false;
@@ -22226,8 +22312,8 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
Resized = true;
GatheredScalars.append(VF - GatheredScalars.size(),
PoisonValue::get(OrigScalarTy));
- NumParts = ::getNumberOfParts(
- *TTI, getWidenedType(OrigScalarTy, VF), OrigScalarTy, VF);
+ NumParts = getNumberOfParts(getWidenedType(OrigScalarTy, VF),
+ OrigScalarTy, VF);
}
}
}
@@ -22464,7 +22550,7 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Type *ScalarTy,
if (Mask.size() == E->Scalars.size())
SliceSize = getPartNumElems(
E->Scalars.size(),
- ::getNumberOfParts(*TTI, VecTy, ScalarTy, E->Scalars.size()));
+ getNumberOfParts(VecTy, ScalarTy, E->Scalars.size()));
SmallVector<int> VecMask(Mask.size(), PoisonMaskElem);
for (const auto [I, TEs] : enumerate(Entries)) {
if (TEs.empty()) {
@@ -25839,6 +25925,16 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
}
}
+ // The lanes cannot depend on each other, so the bundle with the operand of
+ // another member is never scheduled.
+ if (!HasCopyables && any_of(VL, [&](Value *V) {
+ auto *I = dyn_cast<Instruction>(V);
+ return I && any_of(I->operand_values(), [&](const Value *Op) {
+ return is_contained(VL, Op);
+ });
+ }))
+ return std::nullopt;
+
// Initialize the instruction bundle.
Instruction *OldScheduleEnd = ScheduleEnd;
LLVM_DEBUG(dbgs() << "SLP: bundle: " << *S.getMainOp() << "\n");
@@ -27036,7 +27132,7 @@ bool BoUpSLP::collectValuesToDemote(
BitWidth = std::max(BitWidth, BitWidth1);
return BitWidth > 0 && OrigBitWidth >= (BitWidth * 2);
};
- auto FinalAnalysis = [&, TTI = TTI]() {
+ auto FinalAnalysis = [&]() {
if (!IsProfitableToDemote)
return false;
bool Res = all_of(
@@ -27066,10 +27162,8 @@ bool BoUpSLP::collectValuesToDemote(
const unsigned VF = E.Scalars.size();
Type *OrigScalarTy = E.Scalars.front()->getType();
if (UniqueBases.size() <= 2 ||
- ::getNumberOfParts(*TTI, getWidenedType(OrigScalarTy, VF),
- OrigScalarTy) >=
- ::getNumberOfParts(
- *TTI,
+ getNumberOfParts(getWidenedType(OrigScalarTy, VF), OrigScalarTy) >=
+ getNumberOfParts(
getWidenedType(
IntegerType::get(OrigScalarTy->getContext(), BitWidth),
VF),
@@ -27529,7 +27623,7 @@ void BoUpSLP::computeMinimumValueSizes() {
return 0u;
unsigned NumParts =
- ::getNumberOfParts(*TTI, getWidenedType(ScalarTy, VF), ScalarTy);
+ getNumberOfParts(getWidenedType(ScalarTy, VF), ScalarTy);
// The maximum bit width required to represent all the values that can be
// demoted without loss of precision. It would be safe to truncate the roots
@@ -27602,8 +27696,7 @@ void BoUpSLP::computeMinimumValueSizes() {
// use - ignore it.
if (NumParts > 1 &&
NumParts ==
- ::getNumberOfParts(
- *TTI,
+ getNumberOfParts(
getWidenedType(
IntegerType::get(F->getContext(), bit_ceil(MaxBitWidth)),
VF),
@@ -27896,6 +27989,20 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
}
}
+ // Instructions with the single user require just one extract per lane, so
+ // they are used as the seeds for the very last attempt, after all the other
+ // roots in the function are exhausted.
+ if (VectorizeOnceUsed) {
+ for (auto *BB : post_order(&F.getEntryBlock())) {
+ if (BB->isEHPad() ||
+ isa_and_nonnull<UnreachableInst>(BB->getTerminator()) ||
+ R.isScalarFallbackBlock(BB))
+ continue;
+ R.clearReductionData();
+ Changed |= vectorizeOnceUsedSeeds(BB, R);
+ }
+ }
+
if (Changed) {
R.optimizeGatherSequence();
LLVM_DEBUG(dbgs() << "SLP: vectorized \"" << F.getName() << "\"\n");
@@ -28016,6 +28123,21 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
Size = (!IsAllowedSize && S) ? 1 : 2;
return false;
}
+ // Every value is reused several times outside of the chain: such a chain
+ // only merges the stores, while the scalars remain live for the other users
+ // and all the lanes are gathered back. A single outside use may still be a
+ // part of the larger vectorizable graph, same for the values, fed by the
+ // loads, where the vector loads may pay off the gathering.
+ if (S && S.getOpcode() != Instruction::Load &&
+ all_of(ValOps.getArrayRef(), [&](Value *V) {
+ return none_of(cast<Instruction>(V)->operand_values(),
+ IsaPred<LoadInst>) &&
+ count_if(V->users(),
+ [&](User *U) { return !Stores.contains(U); }) > 1;
+ })) {
+ Size = 1;
+ return false;
+ }
}
R.buildTree(Chain);
// Check if tree tiny and store itself or its value is not vectorized.
@@ -28939,7 +29061,7 @@ void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
bool MaxVFOnly,
- bool LimitToRegisterVF) {
+ bool StandaloneSeeds) {
if (VL.size() < 2)
return false;
@@ -28979,8 +29101,8 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
unsigned MaxVF = std::max<unsigned>(
getFloorFullVectorNumberOfElements(*TTI, ScalarTy, VL.size()), MinVF);
MaxVF = std::min(R.getMaximumVF(Sz, S.getOpcode()), MaxVF);
- // Standalone poor-throughput seeds only need one register worth of lanes.
- if (LimitToRegisterVF && Sz != 0)
+ // Standalone seeds only need one register worth of lanes.
+ if (StandaloneSeeds && Sz != 0)
MaxVF = std::min(MaxVF, std::max(MinVF, R.getMaxVecRegSize() / Sz));
if (MaxVF < 2) {
R.getORE()->emit([&]() {
@@ -29017,27 +29139,53 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
SmallVector<Value *> Ops(ActualVF, nullptr);
unsigned Idx = 0;
+ SmallPtrSet<const Value *, 8> Taken;
for (Value *V : VL.drop_front(I)) {
// Check that a previous iteration of this loop did not delete the
// Value.
- if (auto *Inst = dyn_cast<Instruction>(V);
- !Inst || !R.isDeleted(Inst)) {
- Ops[Idx] = V;
- ++Idx;
- if (Idx == ActualVF)
- break;
- }
+ auto *Inst = dyn_cast<Instruction>(V);
+ if (Inst && R.isDeleted(Inst))
+ continue;
+ // The lanes cannot depend on each other, so the seed, fed by the
+ // already taken one, is left for the next window.
+ if (StandaloneSeeds && Inst &&
+ any_of(Inst->operand_values(),
+ [&](const Value *Op) { return Taken.contains(Op); }))
+ continue;
+ Ops[Idx] = V;
+ if (StandaloneSeeds)
+ Taken.insert(V);
+ ++Idx;
+ if (Idx == ActualVF)
+ break;
}
// Not enough vectorizable instructions - exit.
if (Idx != ActualVF)
break;
+ // The identical lanes compute the very same value, so the vector node
+ // only duplicates it, and the window, rejected by another attempt in the
+ // block, is not analyzed again.
+ if (R.isAnalyzedBundle(Ops) ||
+ (StandaloneSeeds && all_of(drop_begin(Ops), [&](Value *V) {
+ return cast<Instruction>(Ops.front())
+ ->isIdenticalTo(cast<Instruction>(V));
+ }))) {
+ if (StandaloneSeeds)
+ I += ActualVF - 1;
+ continue;
+ }
+
LLVM_DEBUG(dbgs() << "SLP: Analyzing " << ActualVF << " operations "
<< "\n");
R.buildTree(Ops);
- if (R.isTreeTinyAndNotFullyVectorizable())
+ if (R.isTreeTinyAndNotFullyVectorizable()) {
+ R.analyzedBundle(Ops);
+ if (StandaloneSeeds)
+ I += ActualVF - 1;
continue;
+ }
if (R.isProfitableToReorder()) {
R.reorderTopToBottom();
R.reorderBottomToTop(
@@ -29057,16 +29205,20 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
if (Cost < -SLPCostThreshold) {
LLVM_DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n");
R.getORE()->emit(OptimizationRemark(SV_NAME, "VectorizedList",
- cast<Instruction>(Ops[0]))
- << "SLP vectorized with cost " << ore::NV("Cost", Cost)
- << " and with tree size "
- << ore::NV("TreeSize", R.getTreeSize()));
+ cast<Instruction>(Ops[0]))
+ << "SLP vectorized with cost " << ore::NV("Cost", Cost)
+ << " and with tree size "
+ << ore::NV("TreeSize", R.getTreeSize()));
R.vectorizeTree();
// Move to the next bundle.
I += VF - 1;
NextInst = I + 1;
Changed = true;
+ } else {
+ R.analyzedBundle(Ops);
+ if (StandaloneSeeds)
+ I += ActualVF - 1;
}
}
}
@@ -29075,8 +29227,8 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
R.getORE()->emit([&]() {
return OptimizationRemarkMissed(SV_NAME, "NotBeneficial", I0)
<< "List vectorization was possible but not beneficial with cost "
- << ore::NV("Cost", MinCost) << " >= "
- << ore::NV("Treshold", -SLPCostThreshold);
+ << ore::NV("Cost", MinCost)
+ << " >= " << ore::NV("Treshold", -SLPCostThreshold);
});
} else if (!Changed) {
R.getORE()->emit([&]() {
@@ -33030,6 +33182,13 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
}
}
+ // The values, feeding only the assumes, and the instructions, producing no
+ // code, are dropped before the codegen, the tree is rejected for them
+ // anyway.
+ if (isAssumeLikeIntrinsic(&*It) || It->isDebugOrPseudoInst() ||
+ R.isEphemeralValue(&*It))
+ continue;
+
if (isa<InsertElementInst, InsertValueInst>(It))
PostProcessInserts.insert(&*It);
else if (auto *CI = dyn_cast<CmpInst>(It))
@@ -33100,56 +33259,113 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
SmallVector<Value *> Seeds;
SmallDenseMap<Value *, SeedGroupKey> SeedKeys;
for (Instruction *I : PoorThroughputSeeds) {
- if (R.isDeleted(I) || !isValidElementType(getValueType(I)))
- continue;
- // Skip seeds a user already resolved: a deleted user was folded into
- // some other vector by an earlier pass, and I's role there is already
- // decided, so retrying I here is almost always a wasted attempt.
- if (any_of(I->users(), [&](User *U) {
- auto *UI = dyn_cast<Instruction>(U);
- return UI && (R.isDeleted(UI) || R.isVectorized(UI));
- }))
+ if (R.isDeleted(I) || !isValidElementType(getValueType(I)) ||
+ hasResolvedUser(I, R))
continue;
SeedKeys.try_emplace(I, getSeedGroupKey(I, *TLI));
Seeds.push_back(I);
}
- auto SeedSorter = [&](Value *V1, Value *V2) {
- if (V1 == V2)
- return false;
- auto *I1 = cast<Instruction>(V1);
- auto *I2 = cast<Instruction>(V2);
- Type *T1 = I1->getType();
- Type *T2 = I2->getType();
- if (T1->getTypeID() != T2->getTypeID())
- return T1->getTypeID() < T2->getTypeID();
- if (T1->getScalarSizeInBits() != T2->getScalarSizeInBits())
- return T1->getScalarSizeInBits() < T2->getScalarSizeInBits();
- const SeedGroupKey &K1 = SeedKeys.at(V1);
- const SeedGroupKey &K2 = SeedKeys.at(V2);
- if (K1 != K2)
- return K1.less(K2);
- return I1->comesBefore(I2);
- };
- auto AreCompatibleSeeds = [&](ArrayRef<Value *> VL, Value *V) {
- if (VL.empty() || VL.back() == V)
- return true;
- return cast<Instruction>(VL.back())->getType() ==
- cast<Instruction>(V)->getType() &&
- SeedKeys.at(VL.back()) == SeedKeys.at(V);
- };
- if (Seeds.size() >= 2)
- Changed |= tryToVectorizeSequence<Value>(
- Seeds, SeedSorter, AreCompatibleSeeds,
- [this, &R](ArrayRef<Value *> Candidates, bool MaxVFOnly) {
- return tryToVectorizeList(Candidates, R, MaxVFOnly,
- /*LimitToRegisterVF=*/true);
- },
- /*MaxVFOnly=*/false, R);
+ Changed |= vectorizeSeeds(
+ Seeds,
+ [&](Value *V1, Value *V2) {
+ return SeedKeys.at(V1).less(SeedKeys.at(V2));
+ },
+ R);
}
return Changed;
}
+bool SLPVectorizerPass::vectorizeSeeds(
+ SmallVectorImpl<Value *> &Seeds,
+ function_ref<bool(Value *, Value *)> IsLessGroup, BoUpSLP &R) {
+ if (Seeds.size() < 2)
+ return false;
+ auto SeedSorter = [&](Value *V1, Value *V2) {
+ if (V1 == V2)
+ return false;
+ auto *I1 = cast<Instruction>(V1);
+ auto *I2 = cast<Instruction>(V2);
+ Type *T1 = I1->getType();
+ Type *T2 = I2->getType();
+ if (T1->getTypeID() != T2->getTypeID())
+ return T1->getTypeID() < T2->getTypeID();
+ if (T1->getScalarSizeInBits() != T2->getScalarSizeInBits())
+ return T1->getScalarSizeInBits() < T2->getScalarSizeInBits();
+ if (IsLessGroup(V1, V2))
+ return true;
+ if (IsLessGroup(V2, V1))
+ return false;
+ return I1->comesBefore(I2);
+ };
+ auto AreCompatibleSeeds = [&](ArrayRef<Value *> VL, Value *V) {
+ if (VL.empty() || VL.back() == V)
+ return true;
+ return cast<Instruction>(VL.back())->getType() ==
+ cast<Instruction>(V)->getType() &&
+ !IsLessGroup(VL.back(), V) && !IsLessGroup(V, VL.back());
+ };
+ return tryToVectorizeSequence<Value>(
+ Seeds, SeedSorter, AreCompatibleSeeds,
+ [this, &R](ArrayRef<Value *> Candidates, bool MaxVFOnly) {
+ // The same list may be retried for the different groups of the same
+ // type, the earlier rejection is still valid for the block.
+ if (R.isAnalyzedBundle(Candidates))
+ return false;
+ if (tryToVectorizeList(Candidates, R, MaxVFOnly,
+ /*StandaloneSeeds=*/true))
+ return true;
+ R.analyzedBundle(Candidates);
+ return false;
+ },
+ /*MaxVFOnly=*/false, R);
+}
+
+bool SLPVectorizerPass::vectorizeOnceUsedSeeds(BasicBlock *BB, BoUpSLP &R) {
+ // Loads are not seeds, their subkeys do not affect the grouping.
+ auto GenerateLoadsSubkey = [](size_t, LoadInst *LI) {
+ return hash_value(LI->getPointerOperand());
+ };
+ SmallVector<Value *> Seeds;
+ SmallDenseMap<std::pair<size_t, size_t>, unsigned> KeyToGroup;
+ SmallDenseMap<Value *, unsigned> SeedGroups;
+ PoorThroughputOpCache PoorThroughputCache;
+ for (Instruction &I : *BB) {
+ if (R.isDeleted(&I) || !I.hasOneUse() || R.isVectorized(&I) ||
+ R.isAnalyzedScalar(&I) || !isValidElementType(getValueType(&I)) ||
+ !isOnceUsedSeed(&I) || isNonVectorizableInst(&I, TLI) ||
+ hasResolvedUser(&I, R))
+ continue;
+ // The poor-throughput ops are seeded on their own, with the different
+ // grouping.
+ if (VectorizePoorThroughput &&
+ isPoorThroughputOp(&I, *TTI, *TLI, PoorThroughputCache))
+ continue;
+ // The multiplication is contracted into the scalar FMA with its user, the
+ // vector node breaks the contraction.
+ if (I.getOpcode() == Instruction::FMul) {
+ auto *U = cast<Instruction>(I.user_back());
+ if (InstructionsState S = getSameOpcode(U, *TLI);
+ S && S.isAddSubLikeOp() &&
+ canConvertToFMA(U, S, *DT, *DL, *TTI, *TLI).isValid())
+ continue;
+ }
+ // The keys are hashes, so the groups are numbered by the first seed to
+ // keep the order deterministic.
+ std::pair<size_t, size_t> Key = generateKeySubkey(
+ &I, TLI, GenerateLoadsSubkey, /*AllowAlternate=*/false);
+ SeedGroups.try_emplace(
+ &I, KeyToGroup.try_emplace(Key, KeyToGroup.size()).first->second);
+ Seeds.push_back(&I);
+ }
+ return vectorizeSeeds(
+ Seeds,
+ [&](Value *V1, Value *V2) {
+ return SeedGroups.at(V1) < SeedGroups.at(V2);
+ },
+ R);
+}
+
bool SLPVectorizerPass::vectorizeGEPIndices(BasicBlock *BB, BoUpSLP &R) {
auto Changed = false;
for (auto &Entry : GEPs) {
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
index 586753e563555..9c55472c1be1e 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
@@ -676,4 +676,58 @@ Intrinsic::ID getMaskedDivRemIntrinsic(unsigned Opcode) {
}
}
+/// Returns true if \p I is a part of a single-use chain, computing an address,
+/// which does not pay off the vectorization: a constant table is accessed by a
+/// gather, while the indices, unrelated between the lanes, require a full
+/// buildvector, unlike the ones, shifted by a constant from a common base.
+static bool isNonProfitableIndex(const Instruction *I) {
+ constexpr unsigned MaxIndexChainLength = 3;
+ // A constant shift of a common base is a cheap buildvector, while the loads
+ // are vectorized together with the indices, computed from them.
+ auto IsProfitableOperand = [](const Value *V) {
+ if (isa<Constant>(V))
+ return true;
+ if (const auto *Cast = dyn_cast<CastInst>(V); Cast && Cast->hasOneUse())
+ V = Cast->getOperand(0);
+ return isa<LoadInst>(V);
+ };
+ const User *U = I->user_back();
+ for ([[maybe_unused]] unsigned _ : seq<unsigned>(MaxIndexChainLength)) {
+ if (const auto *GEP = dyn_cast<GetElementPtrInst>(U))
+ return isa<Constant>(GEP->getPointerOperand()) ||
+ none_of(I->operand_values(), IsProfitableOperand);
+ if (!isa<Instruction>(U) || !U->hasOneUse())
+ return false;
+ U = U->user_back();
+ }
+ return false;
+}
+
+bool isOnceUsedSeed(const Instruction *I) {
+ if (!I->hasOneUse() || isNonProfitableIndex(I))
+ return false;
+ // The operation with the identity or the absorbing constant is folded away
+ // before the codegen, the vector node only repacks the lanes.
+ if (const auto *BO = dyn_cast<BinaryOperator>(I)) {
+ unsigned Opcode = BO->getOpcode();
+ Type *Ty = BO->getType();
+ for (unsigned Idx : seq<unsigned>(2)) {
+ const auto *C = dyn_cast<Constant>(BO->getOperand(Idx));
+ if (C && (C == ConstantExpr::getBinOpIdentity(
+ Opcode, Ty, /*AllowRHSConstant=*/Idx == 1) ||
+ C == ConstantExpr::getBinOpAbsorber(
+ Opcode, Ty, /*AllowLHSConstant=*/Idx == 0)))
+ return false;
+ }
+ }
+ const User *U = I->user_back();
+ if (isa<ExtractElementInst>(I))
+ return isa<InsertElementInst, InsertValueInst>(U);
+ if (isa<CastInst>(I))
+ return !isa<FPToSIInst, FPToUIInst>(I) &&
+ (!isa<CastInst>(U) || U->hasOneUse());
+ return isa<BinaryOperator, UnaryOperator, SelectInst, FreezeInst, CallInst>(
+ I);
+}
+
} // namespace llvm::slpvectorizer
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
index c8a3204649275..14048acb63df9 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
@@ -302,6 +302,16 @@ SmallVector<Constant *> replicateMask(ArrayRef<Constant *> Val, unsigned VF);
/// unlike the plain opcode.
Intrinsic::ID getMaskedDivRemIntrinsic(unsigned Opcode);
+/// Returns true if \p I forms a vectorizable bundle on its own and its single
+/// user does not tear the vector apart. Loads and addresses are excluded: the
+/// tree is built without the users, so it does not pay off the extracts. A
+/// cast, feeding a multi-used cast, is excluded for the same reason, such a
+/// user stays scalar. The fp-to-int conversions move the result to the other
+/// register domain, so the extracts are paid on top of the repacking. The
+/// values, feeding the inserts, are vectorized together with them by the
+/// dedicated attempt.
+bool isOnceUsedSeed(const Instruction *I);
+
} // namespace llvm::slpvectorizer
#endif // LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPUTILS_H
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/avg.ll b/llvm/test/Transforms/PhaseOrdering/X86/avg.ll
index 9084407f008ae..5748cd015dd7e 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/avg.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/avg.ll
@@ -184,67 +184,121 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
; SSE4-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue { i64, i64 } [[DOTFCA_0_INSERT]], i64 [[TMP69]], 1
; SSE4-NEXT: ret { i64, i64 } [[DOTFCA_1_INSERT]]
;
-; AVX-LABEL: @avgr_16_u8(
-; AVX-NEXT: entry:
-; AVX-NEXT: [[TMP0:%.*]] = trunc i64 [[A_COERCE0:%.*]] to i16
-; AVX-NEXT: [[TMP1:%.*]] = lshr i16 [[TMP0]], 8
-; AVX-NEXT: [[TMP2:%.*]] = trunc i64 [[A_COERCE1:%.*]] to i16
-; AVX-NEXT: [[TMP3:%.*]] = lshr i16 [[TMP2]], 8
-; AVX-NEXT: [[TMP4:%.*]] = trunc i64 [[B_COERCE0:%.*]] to i16
-; AVX-NEXT: [[TMP5:%.*]] = lshr i16 [[TMP4]], 8
-; AVX-NEXT: [[TMP6:%.*]] = trunc i64 [[B_COERCE1:%.*]] to i16
-; AVX-NEXT: [[TMP7:%.*]] = lshr i16 [[TMP6]], 8
-; AVX-NEXT: [[CONV1:%.*]] = and i64 [[A_COERCE0]], 255
-; AVX-NEXT: [[CONV4:%.*]] = and i64 [[B_COERCE0]], 255
-; AVX-NEXT: [[ADD:%.*]] = add nuw nsw i64 [[CONV1]], 1
-; AVX-NEXT: [[ADD5:%.*]] = add nuw nsw i64 [[ADD]], [[CONV4]]
-; AVX-NEXT: [[ADD_1:%.*]] = add nuw nsw i16 [[TMP1]], 1
-; AVX-NEXT: [[ADD5_1:%.*]] = add nuw nsw i16 [[ADD_1]], [[TMP5]]
-; AVX-NEXT: [[CONV1_8:%.*]] = and i64 [[A_COERCE1]], 255
-; AVX-NEXT: [[CONV4_8:%.*]] = and i64 [[B_COERCE1]], 255
-; AVX-NEXT: [[ADD_8:%.*]] = add nuw nsw i64 [[CONV1_8]], 1
-; AVX-NEXT: [[ADD5_8:%.*]] = add nuw nsw i64 [[ADD_8]], [[CONV4_8]]
-; AVX-NEXT: [[ADD_9:%.*]] = add nuw nsw i16 [[TMP3]], 1
-; AVX-NEXT: [[ADD5_9:%.*]] = add nuw nsw i16 [[ADD_9]], [[TMP7]]
-; AVX-NEXT: [[TMP8:%.*]] = shl nuw i16 [[ADD5_1]], 7
-; AVX-NEXT: [[TMP9:%.*]] = and i16 [[TMP8]], -256
-; AVX-NEXT: [[TMP10:%.*]] = insertelement <8 x i64> <i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 -1, i64 -1>, i64 [[B_COERCE0]], i64 0
-; AVX-NEXT: [[TMP11:%.*]] = shufflevector <8 x i64> [[TMP10]], <8 x i64> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 6, i32 7>
-; AVX-NEXT: [[TMP12:%.*]] = lshr <8 x i64> [[TMP11]], <i64 56, i64 48, i64 40, i64 32, i64 24, i64 16, i64 0, i64 0>
-; AVX-NEXT: [[RETVAL_SROA_2_0_INSERT_SHIFT_MASKED:%.*]] = zext i16 [[TMP9]] to i64
-; AVX-NEXT: [[TMP14:%.*]] = insertelement <8 x i64> poison, i64 [[A_COERCE0]], i64 0
-; AVX-NEXT: [[TMP15:%.*]] = insertelement <8 x i64> [[TMP14]], i64 [[ADD5]], i64 1
-; AVX-NEXT: [[TMP16:%.*]] = insertelement <8 x i64> [[TMP15]], i64 [[RETVAL_SROA_2_0_INSERT_SHIFT_MASKED]], i64 2
-; AVX-NEXT: [[TMP17:%.*]] = shufflevector <8 x i64> [[TMP16]], <8 x i64> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 2>
-; AVX-NEXT: [[TMP18:%.*]] = lshr <8 x i64> [[TMP17]], <i64 56, i64 48, i64 40, i64 32, i64 24, i64 16, i64 1, i64 0>
-; AVX-NEXT: [[TMP19:%.*]] = and <8 x i64> [[TMP18]], <i64 -1, i64 255, i64 255, i64 255, i64 255, i64 255, i64 -1, i64 -1>
-; AVX-NEXT: [[TMP30:%.*]] = and <8 x i64> [[TMP12]], <i64 -1, i64 255, i64 255, i64 255, i64 255, i64 255, i64 0, i64 0>
-; AVX-NEXT: [[TMP20:%.*]] = add nuw nsw <8 x i64> [[TMP19]], <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 0, i64 0>
-; AVX-NEXT: [[TMP21:%.*]] = add nuw nsw <8 x i64> [[TMP20]], [[TMP30]]
-; AVX-NEXT: [[TMP22:%.*]] = shl nuw <8 x i64> [[TMP21]], <i64 55, i64 47, i64 39, i64 31, i64 23, i64 15, i64 0, i64 0>
-; AVX-NEXT: [[TMP23:%.*]] = and <8 x i64> [[TMP22]], <i64 -72057594037927936, i64 71776119061217280, i64 280375465082880, i64 1095216660480, i64 4278190080, i64 16711680, i64 -1, i64 -1>
-; AVX-NEXT: [[TMP24:%.*]] = tail call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP23]])
-; AVX-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue { i64, i64 } poison, i64 [[TMP24]], 0
-; AVX-NEXT: [[TMP25:%.*]] = shl nuw i16 [[ADD5_9]], 7
-; AVX-NEXT: [[TMP26:%.*]] = and i16 [[TMP25]], -256
-; AVX-NEXT: [[TMP27:%.*]] = insertelement <8 x i64> <i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 -1, i64 -1>, i64 [[B_COERCE1]], i64 0
-; AVX-NEXT: [[TMP28:%.*]] = shufflevector <8 x i64> [[TMP27]], <8 x i64> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 6, i32 7>
-; AVX-NEXT: [[TMP29:%.*]] = lshr <8 x i64> [[TMP28]], <i64 56, i64 48, i64 40, i64 32, i64 24, i64 16, i64 0, i64 0>
-; AVX-NEXT: [[RETVAL_SROA_11_8_INSERT_SHIFT_MASKED:%.*]] = zext i16 [[TMP26]] to i64
-; AVX-NEXT: [[TMP31:%.*]] = insertelement <8 x i64> poison, i64 [[A_COERCE1]], i64 0
-; AVX-NEXT: [[TMP32:%.*]] = insertelement <8 x i64> [[TMP31]], i64 [[ADD5_8]], i64 1
-; AVX-NEXT: [[TMP33:%.*]] = insertelement <8 x i64> [[TMP32]], i64 [[RETVAL_SROA_11_8_INSERT_SHIFT_MASKED]], i64 2
-; AVX-NEXT: [[TMP34:%.*]] = shufflevector <8 x i64> [[TMP33]], <8 x i64> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 2>
-; AVX-NEXT: [[TMP35:%.*]] = lshr <8 x i64> [[TMP34]], <i64 56, i64 48, i64 40, i64 32, i64 24, i64 16, i64 1, i64 0>
-; AVX-NEXT: [[TMP36:%.*]] = and <8 x i64> [[TMP35]], <i64 -1, i64 255, i64 255, i64 255, i64 255, i64 255, i64 -1, i64 -1>
-; AVX-NEXT: [[TMP42:%.*]] = and <8 x i64> [[TMP29]], <i64 -1, i64 255, i64 255, i64 255, i64 255, i64 255, i64 0, i64 0>
-; AVX-NEXT: [[TMP37:%.*]] = add nuw nsw <8 x i64> [[TMP36]], <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 0, i64 0>
-; AVX-NEXT: [[TMP38:%.*]] = add nuw nsw <8 x i64> [[TMP37]], [[TMP42]]
-; AVX-NEXT: [[TMP39:%.*]] = shl nuw <8 x i64> [[TMP38]], <i64 55, i64 47, i64 39, i64 31, i64 23, i64 15, i64 0, i64 0>
-; AVX-NEXT: [[TMP40:%.*]] = and <8 x i64> [[TMP39]], <i64 -72057594037927936, i64 71776119061217280, i64 280375465082880, i64 1095216660480, i64 4278190080, i64 16711680, i64 -1, i64 -1>
-; AVX-NEXT: [[TMP41:%.*]] = tail call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP40]])
-; AVX-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue { i64, i64 } [[DOTFCA_0_INSERT]], i64 [[TMP41]], 1
-; AVX-NEXT: ret { i64, i64 } [[DOTFCA_1_INSERT]]
+; AVX2-LABEL: @avgr_16_u8(
+; AVX2-NEXT: entry:
+; AVX2-NEXT: [[TMP0:%.*]] = trunc i64 [[A_COERCE0:%.*]] to i16
+; AVX2-NEXT: [[TMP1:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i64 0
+; AVX2-NEXT: [[TMP2:%.*]] = trunc i64 [[A_COERCE1:%.*]] to i16
+; AVX2-NEXT: [[TMP3:%.*]] = insertelement <2 x i16> [[TMP1]], i16 [[TMP2]], i64 1
+; AVX2-NEXT: [[TMP4:%.*]] = trunc i64 [[B_COERCE0:%.*]] to i16
+; AVX2-NEXT: [[TMP5:%.*]] = insertelement <2 x i16> poison, i16 [[TMP4]], i64 0
+; AVX2-NEXT: [[TMP6:%.*]] = trunc i64 [[B_COERCE1:%.*]] to i16
+; AVX2-NEXT: [[TMP7:%.*]] = insertelement <2 x i16> [[TMP5]], i16 [[TMP6]], i64 1
+; AVX2-NEXT: [[CONV1:%.*]] = and i64 [[A_COERCE0]], 255
+; AVX2-NEXT: [[CONV4:%.*]] = and i64 [[B_COERCE0]], 255
+; AVX2-NEXT: [[ADD:%.*]] = add nuw nsw i64 [[CONV1]], 1
+; AVX2-NEXT: [[ADD5:%.*]] = add nuw nsw i64 [[ADD]], [[CONV4]]
+; AVX2-NEXT: [[CONV1_8:%.*]] = and i64 [[A_COERCE1]], 255
+; AVX2-NEXT: [[CONV4_8:%.*]] = and i64 [[B_COERCE1]], 255
+; AVX2-NEXT: [[ADD_8:%.*]] = add nuw nsw i64 [[CONV1_8]], 1
+; AVX2-NEXT: [[ADD5_8:%.*]] = add nuw nsw i64 [[ADD_8]], [[CONV4_8]]
+; AVX2-NEXT: [[TMP8:%.*]] = insertelement <8 x i64> <i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 -1, i64 -1>, i64 [[B_COERCE0]], i64 0
+; AVX2-NEXT: [[TMP9:%.*]] = shufflevector <8 x i64> [[TMP8]], <8 x i64> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 6, i32 7>
+; AVX2-NEXT: [[TMP10:%.*]] = lshr <8 x i64> [[TMP9]], <i64 56, i64 48, i64 40, i64 32, i64 24, i64 16, i64 0, i64 0>
+; AVX2-NEXT: [[TMP11:%.*]] = insertelement <8 x i64> poison, i64 [[A_COERCE0]], i64 0
+; AVX2-NEXT: [[TMP12:%.*]] = insertelement <8 x i64> [[TMP11]], i64 [[ADD5]], i64 1
+; AVX2-NEXT: [[TMP13:%.*]] = and <8 x i64> [[TMP10]], <i64 -1, i64 255, i64 255, i64 255, i64 255, i64 255, i64 0, i64 0>
+; AVX2-NEXT: [[TMP14:%.*]] = insertelement <8 x i64> <i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 -1, i64 -1>, i64 [[B_COERCE1]], i64 0
+; AVX2-NEXT: [[TMP15:%.*]] = shufflevector <8 x i64> [[TMP14]], <8 x i64> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 6, i32 7>
+; AVX2-NEXT: [[TMP16:%.*]] = lshr <8 x i64> [[TMP15]], <i64 56, i64 48, i64 40, i64 32, i64 24, i64 16, i64 0, i64 0>
+; AVX2-NEXT: [[TMP17:%.*]] = lshr <2 x i16> [[TMP3]], splat (i16 8)
+; AVX2-NEXT: [[TMP18:%.*]] = lshr <2 x i16> [[TMP7]], splat (i16 8)
+; AVX2-NEXT: [[TMP19:%.*]] = add nuw nsw <2 x i16> [[TMP17]], splat (i16 1)
+; AVX2-NEXT: [[TMP20:%.*]] = add nuw nsw <2 x i16> [[TMP19]], [[TMP18]]
+; AVX2-NEXT: [[TMP21:%.*]] = shl nuw <2 x i16> [[TMP20]], splat (i16 7)
+; AVX2-NEXT: [[TMP22:%.*]] = and <2 x i16> [[TMP21]], splat (i16 -256)
+; AVX2-NEXT: [[TMP23:%.*]] = zext <2 x i16> [[TMP22]] to <2 x i64>
+; AVX2-NEXT: [[TMP24:%.*]] = shufflevector <2 x i64> [[TMP23]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[TMP25:%.*]] = shufflevector <8 x i64> [[TMP12]], <8 x i64> [[TMP24]], <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 8>
+; AVX2-NEXT: [[TMP26:%.*]] = lshr <8 x i64> [[TMP25]], <i64 56, i64 48, i64 40, i64 32, i64 24, i64 16, i64 1, i64 0>
+; AVX2-NEXT: [[TMP27:%.*]] = and <8 x i64> [[TMP26]], <i64 -1, i64 255, i64 255, i64 255, i64 255, i64 255, i64 -1, i64 -1>
+; AVX2-NEXT: [[TMP28:%.*]] = add nuw nsw <8 x i64> [[TMP27]], <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 0, i64 0>
+; AVX2-NEXT: [[TMP29:%.*]] = add nuw nsw <8 x i64> [[TMP28]], [[TMP13]]
+; AVX2-NEXT: [[TMP30:%.*]] = shl nuw <8 x i64> [[TMP29]], <i64 55, i64 47, i64 39, i64 31, i64 23, i64 15, i64 0, i64 0>
+; AVX2-NEXT: [[TMP31:%.*]] = and <8 x i64> [[TMP30]], <i64 -72057594037927936, i64 71776119061217280, i64 280375465082880, i64 1095216660480, i64 4278190080, i64 16711680, i64 -1, i64 -1>
+; AVX2-NEXT: [[TMP32:%.*]] = tail call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP31]])
+; AVX2-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue { i64, i64 } poison, i64 [[TMP32]], 0
+; AVX2-NEXT: [[TMP33:%.*]] = insertelement <8 x i64> poison, i64 [[A_COERCE1]], i64 0
+; AVX2-NEXT: [[TMP34:%.*]] = insertelement <8 x i64> [[TMP33]], i64 [[ADD5_8]], i64 1
+; AVX2-NEXT: [[TMP35:%.*]] = shufflevector <8 x i64> [[TMP34]], <8 x i64> [[TMP24]], <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 9>
+; AVX2-NEXT: [[TMP36:%.*]] = lshr <8 x i64> [[TMP35]], <i64 56, i64 48, i64 40, i64 32, i64 24, i64 16, i64 1, i64 0>
+; AVX2-NEXT: [[TMP37:%.*]] = and <8 x i64> [[TMP36]], <i64 -1, i64 255, i64 255, i64 255, i64 255, i64 255, i64 -1, i64 -1>
+; AVX2-NEXT: [[TMP38:%.*]] = and <8 x i64> [[TMP16]], <i64 -1, i64 255, i64 255, i64 255, i64 255, i64 255, i64 0, i64 0>
+; AVX2-NEXT: [[TMP39:%.*]] = add nuw nsw <8 x i64> [[TMP37]], <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 0, i64 0>
+; AVX2-NEXT: [[TMP40:%.*]] = add nuw nsw <8 x i64> [[TMP39]], [[TMP38]]
+; AVX2-NEXT: [[TMP41:%.*]] = shl nuw <8 x i64> [[TMP40]], <i64 55, i64 47, i64 39, i64 31, i64 23, i64 15, i64 0, i64 0>
+; AVX2-NEXT: [[TMP42:%.*]] = and <8 x i64> [[TMP41]], <i64 -72057594037927936, i64 71776119061217280, i64 280375465082880, i64 1095216660480, i64 4278190080, i64 16711680, i64 -1, i64 -1>
+; AVX2-NEXT: [[TMP43:%.*]] = tail call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP42]])
+; AVX2-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue { i64, i64 } [[DOTFCA_0_INSERT]], i64 [[TMP43]], 1
+; AVX2-NEXT: ret { i64, i64 } [[DOTFCA_1_INSERT]]
+;
+; AVX512-LABEL: @avgr_16_u8(
+; AVX512-NEXT: entry:
+; AVX512-NEXT: [[TMP0:%.*]] = trunc i64 [[A_COERCE0:%.*]] to i16
+; AVX512-NEXT: [[TMP1:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i64 0
+; AVX512-NEXT: [[TMP2:%.*]] = trunc i64 [[A_COERCE1:%.*]] to i16
+; AVX512-NEXT: [[TMP3:%.*]] = insertelement <2 x i16> [[TMP1]], i16 [[TMP2]], i64 1
+; AVX512-NEXT: [[TMP4:%.*]] = trunc i64 [[B_COERCE0:%.*]] to i16
+; AVX512-NEXT: [[TMP5:%.*]] = insertelement <2 x i16> poison, i16 [[TMP4]], i64 0
+; AVX512-NEXT: [[TMP6:%.*]] = trunc i64 [[B_COERCE1:%.*]] to i16
+; AVX512-NEXT: [[TMP7:%.*]] = insertelement <2 x i16> [[TMP5]], i16 [[TMP6]], i64 1
+; AVX512-NEXT: [[CONV1:%.*]] = and i64 [[A_COERCE0]], 255
+; AVX512-NEXT: [[CONV4:%.*]] = and i64 [[B_COERCE0]], 255
+; AVX512-NEXT: [[ADD:%.*]] = add nuw nsw i64 [[CONV1]], 1
+; AVX512-NEXT: [[ADD5:%.*]] = add nuw nsw i64 [[ADD]], [[CONV4]]
+; AVX512-NEXT: [[CONV1_8:%.*]] = and i64 [[A_COERCE1]], 255
+; AVX512-NEXT: [[CONV4_8:%.*]] = and i64 [[B_COERCE1]], 255
+; AVX512-NEXT: [[ADD_8:%.*]] = add nuw nsw i64 [[CONV1_8]], 1
+; AVX512-NEXT: [[ADD5_8:%.*]] = add nuw nsw i64 [[ADD_8]], [[CONV4_8]]
+; AVX512-NEXT: [[TMP8:%.*]] = insertelement <8 x i64> <i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 -1, i64 -1>, i64 [[B_COERCE0]], i64 0
+; AVX512-NEXT: [[TMP9:%.*]] = shufflevector <8 x i64> [[TMP8]], <8 x i64> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 6, i32 7>
+; AVX512-NEXT: [[TMP10:%.*]] = lshr <8 x i64> [[TMP9]], <i64 56, i64 48, i64 40, i64 32, i64 24, i64 16, i64 0, i64 0>
+; AVX512-NEXT: [[TMP11:%.*]] = insertelement <8 x i64> poison, i64 [[A_COERCE0]], i64 0
+; AVX512-NEXT: [[TMP12:%.*]] = insertelement <8 x i64> [[TMP11]], i64 [[ADD5]], i64 1
+; AVX512-NEXT: [[TMP13:%.*]] = and <8 x i64> [[TMP10]], <i64 -1, i64 255, i64 255, i64 255, i64 255, i64 255, i64 0, i64 0>
+; AVX512-NEXT: [[TMP14:%.*]] = insertelement <8 x i64> <i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 -1, i64 -1>, i64 [[B_COERCE1]], i64 0
+; AVX512-NEXT: [[TMP15:%.*]] = shufflevector <8 x i64> [[TMP14]], <8 x i64> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 6, i32 7>
+; AVX512-NEXT: [[TMP16:%.*]] = lshr <8 x i64> [[TMP15]], <i64 56, i64 48, i64 40, i64 32, i64 24, i64 16, i64 0, i64 0>
+; AVX512-NEXT: [[TMP17:%.*]] = lshr <2 x i16> [[TMP3]], splat (i16 8)
+; AVX512-NEXT: [[TMP18:%.*]] = lshr <2 x i16> [[TMP7]], splat (i16 8)
+; AVX512-NEXT: [[TMP19:%.*]] = add nuw nsw <2 x i16> [[TMP17]], splat (i16 1)
+; AVX512-NEXT: [[TMP20:%.*]] = add nuw nsw <2 x i16> [[TMP19]], [[TMP18]]
+; AVX512-NEXT: [[TMP21:%.*]] = shl nuw <2 x i16> [[TMP20]], splat (i16 7)
+; AVX512-NEXT: [[TMP22:%.*]] = and <2 x i16> [[TMP21]], splat (i16 -256)
+; AVX512-NEXT: [[TMP23:%.*]] = shufflevector <2 x i16> [[TMP22]], <2 x i16> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT: [[TMP24:%.*]] = zext <8 x i16> [[TMP23]] to <8 x i64>
+; AVX512-NEXT: [[TMP25:%.*]] = shufflevector <8 x i64> [[TMP12]], <8 x i64> [[TMP24]], <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 8>
+; AVX512-NEXT: [[TMP26:%.*]] = lshr <8 x i64> [[TMP25]], <i64 56, i64 48, i64 40, i64 32, i64 24, i64 16, i64 1, i64 0>
+; AVX512-NEXT: [[TMP27:%.*]] = and <8 x i64> [[TMP26]], <i64 -1, i64 255, i64 255, i64 255, i64 255, i64 255, i64 -1, i64 -1>
+; AVX512-NEXT: [[TMP28:%.*]] = add nuw nsw <8 x i64> [[TMP27]], <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 0, i64 0>
+; AVX512-NEXT: [[TMP29:%.*]] = add nuw nsw <8 x i64> [[TMP28]], [[TMP13]]
+; AVX512-NEXT: [[TMP30:%.*]] = shl nuw <8 x i64> [[TMP29]], <i64 55, i64 47, i64 39, i64 31, i64 23, i64 15, i64 0, i64 0>
+; AVX512-NEXT: [[TMP31:%.*]] = and <8 x i64> [[TMP30]], <i64 -72057594037927936, i64 71776119061217280, i64 280375465082880, i64 1095216660480, i64 4278190080, i64 16711680, i64 -1, i64 -1>
+; AVX512-NEXT: [[TMP32:%.*]] = tail call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP31]])
+; AVX512-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue { i64, i64 } poison, i64 [[TMP32]], 0
+; AVX512-NEXT: [[TMP33:%.*]] = insertelement <8 x i64> poison, i64 [[A_COERCE1]], i64 0
+; AVX512-NEXT: [[TMP34:%.*]] = insertelement <8 x i64> [[TMP33]], i64 [[ADD5_8]], i64 1
+; AVX512-NEXT: [[TMP35:%.*]] = shufflevector <8 x i64> [[TMP34]], <8 x i64> [[TMP24]], <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 9>
+; AVX512-NEXT: [[TMP36:%.*]] = lshr <8 x i64> [[TMP35]], <i64 56, i64 48, i64 40, i64 32, i64 24, i64 16, i64 1, i64 0>
+; AVX512-NEXT: [[TMP37:%.*]] = and <8 x i64> [[TMP36]], <i64 -1, i64 255, i64 255, i64 255, i64 255, i64 255, i64 -1, i64 -1>
+; AVX512-NEXT: [[TMP38:%.*]] = and <8 x i64> [[TMP16]], <i64 -1, i64 255, i64 255, i64 255, i64 255, i64 255, i64 0, i64 0>
+; AVX512-NEXT: [[TMP39:%.*]] = add nuw nsw <8 x i64> [[TMP37]], <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 0, i64 0>
+; AVX512-NEXT: [[TMP40:%.*]] = add nuw nsw <8 x i64> [[TMP39]], [[TMP38]]
+; AVX512-NEXT: [[TMP41:%.*]] = shl nuw <8 x i64> [[TMP40]], <i64 55, i64 47, i64 39, i64 31, i64 23, i64 15, i64 0, i64 0>
+; AVX512-NEXT: [[TMP42:%.*]] = and <8 x i64> [[TMP41]], <i64 -72057594037927936, i64 71776119061217280, i64 280375465082880, i64 1095216660480, i64 4278190080, i64 16711680, i64 -1, i64 -1>
+; AVX512-NEXT: [[TMP43:%.*]] = tail call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP42]])
+; AVX512-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue { i64, i64 } [[DOTFCA_0_INSERT]], i64 [[TMP43]], 1
+; AVX512-NEXT: ret { i64, i64 } [[DOTFCA_1_INSERT]]
;
entry:
%retval = alloca %"struct.std::array16", align 1
@@ -791,19 +845,9 @@ define { i64, i64 } @avgr_8_u16_alt(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerc
; SSE2-NEXT: [[A_SROA_4_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0:%.*]], 48
; SSE2-NEXT: [[A_SROA_3_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 32
; SSE2-NEXT: [[A_SROA_2_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 16
-; SSE2-NEXT: [[B_SROA_0_0_EXTRACT_TRUNC:%.*]] = trunc nuw i64 [[A_SROA_4_0_EXTRACT_SHIFT]] to i16
-; SSE2-NEXT: [[B_SROA_2_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_SROA_3_0_EXTRACT_SHIFT]] to i16
; SSE2-NEXT: [[B_SROA_4_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0:%.*]], 48
; SSE2-NEXT: [[B_SROA_3_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 32
; SSE2-NEXT: [[B_SROA_2_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 16
-; SSE2-NEXT: [[B_SROA_4_0_EXTRACT_TRUNC:%.*]] = trunc nuw i64 [[B_SROA_4_0_EXTRACT_SHIFT]] to i16
-; SSE2-NEXT: [[B_SROA_3_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_3_0_EXTRACT_SHIFT]] to i16
-; SSE2-NEXT: [[SHR5:%.*]] = lshr i16 [[B_SROA_0_0_EXTRACT_TRUNC]], 1
-; SSE2-NEXT: [[SHR5_1:%.*]] = lshr i16 [[B_SROA_2_0_EXTRACT_TRUNC]], 1
-; SSE2-NEXT: [[SHR5_3:%.*]] = lshr i16 [[B_SROA_4_0_EXTRACT_TRUNC]], 1
-; SSE2-NEXT: [[SHR5_2:%.*]] = lshr i16 [[B_SROA_3_0_EXTRACT_TRUNC]], 1
-; SSE2-NEXT: [[NARROW:%.*]] = add nuw i16 [[SHR5_3]], [[SHR5]]
-; SSE2-NEXT: [[NARROW_1:%.*]] = add nuw i16 [[SHR5_2]], [[SHR5_1]]
; SSE2-NEXT: [[TMP0:%.*]] = trunc i64 [[A_COERCE0]] to i16
; SSE2-NEXT: [[TMP14:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i64 0
; SSE2-NEXT: [[TMP17:%.*]] = trunc i64 [[A_SROA_2_0_EXTRACT_SHIFT]] to i16
@@ -816,35 +860,43 @@ define { i64, i64 } @avgr_8_u16_alt(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerc
; SSE2-NEXT: [[TMP21:%.*]] = lshr <2 x i16> [[TMP7]], splat (i16 1)
; SSE2-NEXT: [[TMP10:%.*]] = add nuw <2 x i16> [[TMP21]], [[TMP19]]
; SSE2-NEXT: [[TMP37:%.*]] = shufflevector <2 x i16> [[TMP7]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT: [[TMP1:%.*]] = shufflevector <2 x i16> [[TMP18]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT: [[TMP44:%.*]] = shufflevector <2 x i16> [[TMP10]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT: [[A_SROA_9_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1:%.*]], 48
+; SSE2-NEXT: [[A_SROA_8_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 32
+; SSE2-NEXT: [[A_SROA_7_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 16
+; SSE2-NEXT: [[B_SROA_9_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE2:%.*]], 48
+; SSE2-NEXT: [[B_SROA_8_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE2]], 32
+; SSE2-NEXT: [[B_SROA_7_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE2]], 16
+; SSE2-NEXT: [[B_SROA_9_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_8_8_EXTRACT_SHIFT]] to i16
+; SSE2-NEXT: [[B_SROA_8_8_EXTRACT_TRUNC:%.*]] = trunc nuw i64 [[B_SROA_9_8_EXTRACT_SHIFT]] to i16
+; SSE2-NEXT: [[B_SROA_3_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_3_0_EXTRACT_SHIFT]] to i16
+; SSE2-NEXT: [[B_SROA_4_0_EXTRACT_TRUNC:%.*]] = trunc nuw i64 [[B_SROA_4_0_EXTRACT_SHIFT]] to i16
; SSE2-NEXT: [[TMP38:%.*]] = insertelement <4 x i16> [[TMP37]], i16 [[B_SROA_3_0_EXTRACT_TRUNC]], i64 2
; SSE2-NEXT: [[TMP39:%.*]] = insertelement <4 x i16> [[TMP38]], i16 [[B_SROA_4_0_EXTRACT_TRUNC]], i64 3
-; SSE2-NEXT: [[TMP1:%.*]] = shufflevector <2 x i16> [[TMP18]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; SSE2-NEXT: [[NARROW:%.*]] = trunc i64 [[A_SROA_8_8_EXTRACT_SHIFT]] to i16
+; SSE2-NEXT: [[A_SROA_5_8_EXTRACT_TRUNC:%.*]] = trunc nuw i64 [[A_SROA_9_8_EXTRACT_SHIFT]] to i16
+; SSE2-NEXT: [[B_SROA_2_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_SROA_3_0_EXTRACT_SHIFT]] to i16
+; SSE2-NEXT: [[B_SROA_0_0_EXTRACT_TRUNC:%.*]] = trunc nuw i64 [[A_SROA_4_0_EXTRACT_SHIFT]] to i16
; SSE2-NEXT: [[TMP2:%.*]] = insertelement <4 x i16> [[TMP1]], i16 [[B_SROA_2_0_EXTRACT_TRUNC]], i64 2
; SSE2-NEXT: [[TMP3:%.*]] = insertelement <4 x i16> [[TMP2]], i16 [[B_SROA_0_0_EXTRACT_TRUNC]], i64 3
; SSE2-NEXT: [[TMP8:%.*]] = or <4 x i16> [[TMP39]], [[TMP3]]
; SSE2-NEXT: [[TMP9:%.*]] = and <4 x i16> [[TMP8]], splat (i16 1)
-; SSE2-NEXT: [[TMP11:%.*]] = shufflevector <2 x i16> [[TMP10]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; SSE2-NEXT: [[TMP12:%.*]] = insertelement <4 x i16> [[TMP11]], i16 [[NARROW_1]], i64 2
+; SSE2-NEXT: [[TMP48:%.*]] = insertelement <4 x i16> poison, i16 [[B_SROA_0_0_EXTRACT_TRUNC]], i64 0
+; SSE2-NEXT: [[TMP49:%.*]] = insertelement <4 x i16> [[TMP48]], i16 [[A_SROA_5_8_EXTRACT_TRUNC]], i64 1
+; SSE2-NEXT: [[TMP12:%.*]] = insertelement <4 x i16> [[TMP49]], i16 [[B_SROA_2_0_EXTRACT_TRUNC]], i64 2
; SSE2-NEXT: [[TMP13:%.*]] = insertelement <4 x i16> [[TMP12]], i16 [[NARROW]], i64 3
-; SSE2-NEXT: [[TMP15:%.*]] = add nuw <4 x i16> [[TMP13]], [[TMP9]]
+; SSE2-NEXT: [[TMP50:%.*]] = lshr <4 x i16> [[TMP13]], splat (i16 1)
+; SSE2-NEXT: [[TMP51:%.*]] = insertelement <4 x i16> poison, i16 [[B_SROA_4_0_EXTRACT_TRUNC]], i64 0
+; SSE2-NEXT: [[TMP52:%.*]] = insertelement <4 x i16> [[TMP51]], i16 [[B_SROA_8_8_EXTRACT_TRUNC]], i64 1
+; SSE2-NEXT: [[TMP53:%.*]] = insertelement <4 x i16> [[TMP52]], i16 [[B_SROA_3_0_EXTRACT_TRUNC]], i64 2
+; SSE2-NEXT: [[TMP54:%.*]] = insertelement <4 x i16> [[TMP53]], i16 [[B_SROA_9_8_EXTRACT_TRUNC]], i64 3
+; SSE2-NEXT: [[TMP55:%.*]] = lshr <4 x i16> [[TMP54]], splat (i16 1)
+; SSE2-NEXT: [[TMP56:%.*]] = add nuw <4 x i16> [[TMP55]], [[TMP50]]
+; SSE2-NEXT: [[TMP57:%.*]] = shufflevector <4 x i16> [[TMP44]], <4 x i16> [[TMP56]], <4 x i32> <i32 0, i32 1, i32 6, i32 4>
+; SSE2-NEXT: [[TMP15:%.*]] = add nuw <4 x i16> [[TMP57]], [[TMP9]]
; SSE2-NEXT: [[TMP16:%.*]] = bitcast <4 x i16> [[TMP15]] to i64
; SSE2-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue { i64, i64 } poison, i64 [[TMP16]], 0
-; SSE2-NEXT: [[A_SROA_9_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1:%.*]], 48
-; SSE2-NEXT: [[B_SROA_8_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 32
-; SSE2-NEXT: [[A_SROA_7_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 16
-; SSE2-NEXT: [[A_SROA_5_8_EXTRACT_TRUNC:%.*]] = trunc nuw i64 [[A_SROA_9_8_EXTRACT_SHIFT]] to i16
-; SSE2-NEXT: [[A_SROA_7_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_8_8_EXTRACT_SHIFT]] to i16
-; SSE2-NEXT: [[B_SROA_9_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE2:%.*]], 48
-; SSE2-NEXT: [[B_SROA_8_8_EXTRACT_SHIFT1:%.*]] = lshr i64 [[B_COERCE2]], 32
-; SSE2-NEXT: [[B_SROA_7_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE2]], 16
-; SSE2-NEXT: [[B_SROA_8_8_EXTRACT_TRUNC:%.*]] = trunc nuw i64 [[B_SROA_9_8_EXTRACT_SHIFT]] to i16
-; SSE2-NEXT: [[B_SROA_9_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_8_8_EXTRACT_SHIFT1]] to i16
-; SSE2-NEXT: [[SHR_6:%.*]] = lshr i16 [[A_SROA_5_8_EXTRACT_TRUNC]], 1
-; SSE2-NEXT: [[SHR_7:%.*]] = lshr i16 [[A_SROA_7_8_EXTRACT_TRUNC]], 1
-; SSE2-NEXT: [[SHR5_6:%.*]] = lshr i16 [[B_SROA_8_8_EXTRACT_TRUNC]], 1
-; SSE2-NEXT: [[SHR5_7:%.*]] = lshr i16 [[B_SROA_9_8_EXTRACT_TRUNC]], 1
-; SSE2-NEXT: [[NARROW_6:%.*]] = add nuw i16 [[SHR5_6]], [[SHR_6]]
-; SSE2-NEXT: [[NARROW_7:%.*]] = add nuw i16 [[SHR5_7]], [[SHR_7]]
; SSE2-NEXT: [[TMP40:%.*]] = trunc i64 [[B_COERCE1]] to i16
; SSE2-NEXT: [[TMP41:%.*]] = insertelement <2 x i16> poison, i16 [[TMP40]], i64 0
; SSE2-NEXT: [[TMP42:%.*]] = trunc i64 [[A_SROA_7_8_EXTRACT_SHIFT]] to i16
@@ -860,13 +912,12 @@ define { i64, i64 } @avgr_8_u16_alt(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerc
; SSE2-NEXT: [[TMP36:%.*]] = insertelement <4 x i16> [[TMP35]], i16 [[B_SROA_9_8_EXTRACT_TRUNC]], i64 2
; SSE2-NEXT: [[TMP20:%.*]] = insertelement <4 x i16> [[TMP36]], i16 [[B_SROA_8_8_EXTRACT_TRUNC]], i64 3
; SSE2-NEXT: [[TMP22:%.*]] = shufflevector <2 x i16> [[TMP27]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; SSE2-NEXT: [[TMP23:%.*]] = insertelement <4 x i16> [[TMP22]], i16 [[A_SROA_7_8_EXTRACT_TRUNC]], i64 2
+; SSE2-NEXT: [[TMP23:%.*]] = insertelement <4 x i16> [[TMP22]], i16 [[NARROW]], i64 2
; SSE2-NEXT: [[TMP24:%.*]] = insertelement <4 x i16> [[TMP23]], i16 [[A_SROA_5_8_EXTRACT_TRUNC]], i64 3
; SSE2-NEXT: [[TMP25:%.*]] = or <4 x i16> [[TMP20]], [[TMP24]]
; SSE2-NEXT: [[TMP26:%.*]] = and <4 x i16> [[TMP25]], splat (i16 1)
; SSE2-NEXT: [[TMP43:%.*]] = shufflevector <2 x i16> [[TMP34]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; SSE2-NEXT: [[TMP44:%.*]] = insertelement <4 x i16> [[TMP43]], i16 [[NARROW_7]], i64 2
-; SSE2-NEXT: [[TMP30:%.*]] = insertelement <4 x i16> [[TMP44]], i16 [[NARROW_6]], i64 3
+; SSE2-NEXT: [[TMP30:%.*]] = shufflevector <4 x i16> [[TMP43]], <4 x i16> [[TMP56]], <4 x i32> <i32 0, i32 1, i32 7, i32 5>
; SSE2-NEXT: [[TMP31:%.*]] = add nuw <4 x i16> [[TMP30]], [[TMP26]]
; SSE2-NEXT: [[TMP33:%.*]] = bitcast <4 x i16> [[TMP31]] to i64
; SSE2-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue { i64, i64 } [[DOTFCA_0_INSERT]], i64 [[TMP33]], 1
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/extractelements-to-shuffle.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/extractelements-to-shuffle.ll
index b1817688f89a2..b4cffaa21b14a 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/extractelements-to-shuffle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/extractelements-to-shuffle.ll
@@ -75,21 +75,19 @@ define void @dist_vec(ptr nocapture noundef readonly %pA, ptr nocapture noundef
; CHECK-NEXT: [[TMP4TT_0_LCSSA:%.*]] = phi <2 x i64> [ zeroinitializer, [[ENTRY]] ], [ [[ADD_I]], [[WHILE_END_LOOPEXIT]] ]
; CHECK-NEXT: [[PB_ADDR_0_LCSSA:%.*]] = phi ptr [ [[PB]], [[ENTRY]] ], [ [[SCEVGEP311]], [[WHILE_END_LOOPEXIT]] ]
; CHECK-NEXT: [[PA_ADDR_0_LCSSA:%.*]] = phi ptr [ [[PA]], [[ENTRY]] ], [ [[SCEVGEP]], [[WHILE_END_LOOPEXIT]] ]
-; CHECK-NEXT: [[VGETQ_LANE:%.*]] = extractelement <2 x i64> [[TMP4TT_0_LCSSA]], i64 0
-; CHECK-NEXT: [[VGETQ_LANE45:%.*]] = extractelement <2 x i64> [[TMP4TT_0_LCSSA]], i64 1
-; CHECK-NEXT: [[ADD:%.*]] = add i64 [[VGETQ_LANE]], [[VGETQ_LANE45]]
+; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <2 x i64> [[TMP4TT_0_LCSSA]], <2 x i64> [[TMP4FF_0_LCSSA]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <2 x i64> [[TMP4TT_0_LCSSA]], <2 x i64> [[TMP4FF_0_LCSSA]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT: [[TMP24:%.*]] = add <2 x i64> [[TMP22]], [[TMP23]]
+; CHECK-NEXT: [[ADD:%.*]] = extractelement <2 x i64> [[TMP24]], i64 0
; CHECK-NEXT: [[CONV48:%.*]] = trunc i64 [[ADD]] to i32
-; CHECK-NEXT: [[VGETQ_LANE51:%.*]] = extractelement <2 x i64> [[TMP4FF_0_LCSSA]], i64 0
-; CHECK-NEXT: [[VGETQ_LANE55:%.*]] = extractelement <2 x i64> [[TMP4FF_0_LCSSA]], i64 1
-; CHECK-NEXT: [[ADD57:%.*]] = add i64 [[VGETQ_LANE51]], [[VGETQ_LANE55]]
+; CHECK-NEXT: [[ADD57:%.*]] = extractelement <2 x i64> [[TMP24]], i64 1
; CHECK-NEXT: [[CONV60:%.*]] = trunc i64 [[ADD57]] to i32
-; CHECK-NEXT: [[VGETQ_LANE63:%.*]] = extractelement <2 x i64> [[TMP4TF_0_LCSSA]], i64 0
-; CHECK-NEXT: [[VGETQ_LANE67:%.*]] = extractelement <2 x i64> [[TMP4TF_0_LCSSA]], i64 1
-; CHECK-NEXT: [[ADD69:%.*]] = add i64 [[VGETQ_LANE63]], [[VGETQ_LANE67]]
+; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <2 x i64> [[TMP4TF_0_LCSSA]], <2 x i64> [[TMP4FT_0_LCSSA]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: [[TMP26:%.*]] = shufflevector <2 x i64> [[TMP4TF_0_LCSSA]], <2 x i64> [[TMP4FT_0_LCSSA]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT: [[TMP27:%.*]] = add <2 x i64> [[TMP25]], [[TMP26]]
+; CHECK-NEXT: [[ADD69:%.*]] = extractelement <2 x i64> [[TMP27]], i64 0
; CHECK-NEXT: [[CONV72:%.*]] = trunc i64 [[ADD69]] to i32
-; CHECK-NEXT: [[VGETQ_LANE75:%.*]] = extractelement <2 x i64> [[TMP4FT_0_LCSSA]], i64 0
-; CHECK-NEXT: [[VGETQ_LANE79:%.*]] = extractelement <2 x i64> [[TMP4FT_0_LCSSA]], i64 1
-; CHECK-NEXT: [[ADD81:%.*]] = add i64 [[VGETQ_LANE75]], [[VGETQ_LANE79]]
+; CHECK-NEXT: [[ADD81:%.*]] = extractelement <2 x i64> [[TMP27]], i64 1
; CHECK-NEXT: [[CONV84:%.*]] = trunc i64 [[ADD81]] to i32
; CHECK-NEXT: [[AND:%.*]] = and i32 [[NUMBEROFBOOLS]], 127
; CHECK-NEXT: [[CMP86284:%.*]] = icmp ugt i32 [[AND]], 31
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/unsigned-after-sext-node.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/unsigned-after-sext-node.ll
index 96ed3e77d9877..6264054f130bd 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/unsigned-after-sext-node.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/unsigned-after-sext-node.ll
@@ -4,12 +4,8 @@
define i16 @test() {
; CHECK-LABEL: define i16 @test() {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[LNOT:%.*]] = xor i1 true, true
-; CHECK-NEXT: [[LNOT_EXT:%.*]] = zext i1 [[LNOT]] to i16
-; CHECK-NEXT: [[ADD:%.*]] = add nsw i16 0, [[LNOT_EXT]]
-; CHECK-NEXT: [[LNOT5:%.*]] = xor i1 true, true
-; CHECK-NEXT: [[LNOT_EXT6:%.*]] = zext i1 [[LNOT5]] to i16
-; CHECK-NEXT: [[ADD7:%.*]] = add nsw i16 [[ADD]], [[LNOT_EXT6]]
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i16 0, 0
+; CHECK-NEXT: [[ADD7:%.*]] = add nsw i16 [[ADD]], 0
; CHECK-NEXT: ret i16 [[ADD7]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/vectorize-free-extracts-inserts.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/vectorize-free-extracts-inserts.ll
index 6e9d9acbe83b0..0b9717f2bc8b0 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/vectorize-free-extracts-inserts.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/vectorize-free-extracts-inserts.ll
@@ -397,24 +397,24 @@ define void @first_mul_chain_jumbled(ptr %ptr.1, ptr %ptr.2) {
; CHECK-LABEL: @first_mul_chain_jumbled(
; CHECK-NEXT: bb:
; CHECK-NEXT: [[V_1:%.*]] = load <9 x double>, ptr [[PTR_1:%.*]], align 8
-; CHECK-NEXT: [[V1_LANE_2:%.*]] = extractelement <9 x double> [[V_1]], i32 2
-; CHECK-NEXT: [[V1_LANE_5:%.*]] = extractelement <9 x double> [[V_1]], i32 5
; CHECK-NEXT: [[V_2:%.*]] = load <4 x double>, ptr [[PTR_2:%.*]], align 16
-; CHECK-NEXT: [[V2_LANE_0:%.*]] = extractelement <4 x double> [[V_2]], i32 0
-; CHECK-NEXT: [[V2_LANE_1:%.*]] = extractelement <4 x double> [[V_2]], i32 1
; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <8 x i32> <i32 4, i32 3, i32 6, i32 5, i32 8, i32 7, i32 1, i32 0>
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <8 x i32> <i32 1, i32 0, i32 2, i32 0, i32 2, i32 1, i32 0, i32 2>
; CHECK-NEXT: [[TMP2:%.*]] = fmul <8 x double> [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[A_LANE_8:%.*]] = fmul double [[V1_LANE_2]], [[V2_LANE_1]]
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison>
; CHECK-NEXT: [[A_INS_72:%.*]] = shufflevector <9 x double> zeroinitializer, <9 x double> [[TMP3]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>
-; CHECK-NEXT: [[A_INS_8:%.*]] = insertelement <9 x double> [[A_INS_72]], double [[A_LANE_8]], i32 8
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <8 x i32> <i32 6, i32 7, i32 8, i32 0, i32 1, i32 2, i32 3, i32 4>
; CHECK-NEXT: [[TMP5:%.*]] = fmul <8 x double> [[TMP4]], [[TMP1]]
-; CHECK-NEXT: [[B_LANE_8:%.*]] = fmul double [[V1_LANE_5]], [[V2_LANE_0]]
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <9 x double> [[V_1]], <9 x double> poison, <2 x i32> <i32 5, i32 2>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x double> [[V_2]], <4 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP8:%.*]] = fmul <2 x double> [[TMP11]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> poison, <9 x i32> <i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> poison, <9 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[A_INS_8:%.*]] = shufflevector <9 x double> [[TMP3]], <9 x double> [[TMP10]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10>
; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x double> [[TMP5]], <8 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison>
; CHECK-NEXT: [[B_INS_71:%.*]] = shufflevector <9 x double> zeroinitializer, <9 x double> [[TMP6]], <9 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 8>
-; CHECK-NEXT: [[B_INS_8:%.*]] = insertelement <9 x double> [[B_INS_71]], double [[B_LANE_8]], i32 8
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> poison, <9 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[B_INS_8:%.*]] = shufflevector <9 x double> [[TMP6]], <9 x double> [[TMP10]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 9>
; CHECK-NEXT: [[RES:%.*]] = fsub <9 x double> [[A_INS_8]], [[B_INS_8]]
; CHECK-NEXT: store <9 x double> [[RES]], ptr [[PTR_1]], align 8
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/revec.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/revec.ll
index 3025c1085d4e7..86b2acdc7634f 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/revec.ll
@@ -147,19 +147,14 @@ define ptr @test4() {
; POWEROF2-NEXT: [[TMP13:%.*]] = fmul <2 x float> [[TMP12]], zeroinitializer
; POWEROF2-NEXT: [[TMP14:%.*]] = shufflevector <4 x float> [[TMP10]], <4 x float> poison, <2 x i32> <i32 2, i32 3>
; POWEROF2-NEXT: [[TMP15:%.*]] = fmul <2 x float> zeroinitializer, [[TMP14]]
-; POWEROF2-NEXT: [[TMP30:%.*]] = extractelement <2 x float> [[TMP9]], i64 0
-; POWEROF2-NEXT: [[TMP17:%.*]] = fmul float 0.000000e+00, [[TMP30]]
-; POWEROF2-NEXT: [[TMP18:%.*]] = extractelement <2 x float> [[TMP9]], i64 1
-; POWEROF2-NEXT: [[TMP19:%.*]] = fmul float [[TMP18]], 0.000000e+00
-; POWEROF2-NEXT: [[TMP20:%.*]] = extractelement <2 x float> [[TMP13]], i64 0
-; POWEROF2-NEXT: [[TMP21:%.*]] = fadd reassoc nsz float [[TMP20]], [[TMP17]]
-; POWEROF2-NEXT: [[TMP22:%.*]] = extractelement <2 x float> [[TMP15]], i64 0
-; POWEROF2-NEXT: [[TMP23:%.*]] = fadd reassoc nsz float [[TMP22]], [[TMP19]]
-; POWEROF2-NEXT: [[TMP24:%.*]] = extractelement <2 x float> [[TMP13]], i64 1
-; POWEROF2-NEXT: [[TMP25:%.*]] = fadd reassoc nsz float [[TMP21]], [[TMP24]]
-; POWEROF2-NEXT: [[TMP26:%.*]] = extractelement <2 x float> [[TMP15]], i64 1
-; POWEROF2-NEXT: [[TMP27:%.*]] = fadd reassoc nsz float [[TMP23]], [[TMP26]]
+; POWEROF2-NEXT: [[TMP17:%.*]] = fmul <2 x float> zeroinitializer, [[TMP9]]
+; POWEROF2-NEXT: [[TMP18:%.*]] = shufflevector <2 x float> [[TMP13]], <2 x float> [[TMP15]], <2 x i32> <i32 0, i32 2>
+; POWEROF2-NEXT: [[TMP19:%.*]] = fadd reassoc nsz <2 x float> [[TMP18]], [[TMP17]]
+; POWEROF2-NEXT: [[TMP20:%.*]] = shufflevector <2 x float> [[TMP13]], <2 x float> [[TMP15]], <2 x i32> <i32 1, i32 3>
+; POWEROF2-NEXT: [[TMP21:%.*]] = fadd reassoc nsz <2 x float> [[TMP19]], [[TMP20]]
+; POWEROF2-NEXT: [[TMP25:%.*]] = extractelement <2 x float> [[TMP21]], i64 0
; POWEROF2-NEXT: [[TMP28:%.*]] = tail call float @llvm.sqrt.f32(float [[TMP25]])
+; POWEROF2-NEXT: [[TMP27:%.*]] = extractelement <2 x float> [[TMP21]], i64 1
; POWEROF2-NEXT: [[TMP29:%.*]] = tail call float @llvm.sqrt.f32(float [[TMP27]])
; POWEROF2-NEXT: ret ptr null
;
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/runtime-strided-stores.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/runtime-strided-stores.ll
index 397c45192a526..6eb4ddc12ef50 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/runtime-strided-stores.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/runtime-strided-stores.ll
@@ -936,14 +936,14 @@ define void @runtime_stride_diff_types(ptr %pl, ptr %ps, i64 %stride) {
; CHECK-NEXT: [[GEP_S7:%.*]] = getelementptr i8, ptr [[GEP_S6]], i64 [[STRIDE]]
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i8> [[TMP2]], i64 0
; CHECK-NEXT: store i8 [[TMP4]], ptr [[GEP_S0]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i8> [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i8> [[TMP2]], i64 2
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i8> [[TMP2]], i64 3
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[TMP3]], i64 0
; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i16
; CHECK-NEXT: store i16 [[TMP6]], ptr [[GEP_S1]], align 2
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i8> [[TMP2]], i64 1
; CHECK-NEXT: store i8 [[TMP7]], ptr [[GEP_S2]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i8> [[TMP2]], i64 2
; CHECK-NEXT: store i8 [[TMP8]], ptr [[GEP_S3]], align 1
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i8> [[TMP2]], i64 3
; CHECK-NEXT: store i8 [[TMP9]], ptr [[GEP_S4]], align 1
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i8> [[TMP3]], i64 1
; CHECK-NEXT: [[TMP11:%.*]] = zext i8 [[TMP10]] to i16
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll
index 49ee6b303b276..ae1d913ec9c5a 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll
@@ -8,20 +8,20 @@
define <8 x float> @ceil_floor(<8 x float> %a) {
; SSE-LABEL: @ceil_floor(
-; SSE-NEXT: [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i32 0
-; SSE-NEXT: [[A3:%.*]] = extractelement <8 x float> [[A]], i32 3
-; SSE-NEXT: [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; SSE-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
+; SSE-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
; SSE-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP1]])
-; SSE-NEXT: [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
+; SSE-NEXT: [[TMP10:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 0, i32 3>
+; SSE-NEXT: [[TMP13:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP10]])
; SSE-NEXT: [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
; SSE-NEXT: [[TMP4:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
; SSE-NEXT: [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
; SSE-NEXT: [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
-; SSE-NEXT: [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i32 0
+; SSE-NEXT: [[R0:%.*]] = shufflevector <2 x float> [[TMP13]], <2 x float> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; SSE-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; SSE-NEXT: [[R2:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE-NEXT: [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i32 3
+; SSE-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> [[TMP13]], <2 x float> poison, <8 x i32> <i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT: [[TMP12:%.*]] = shufflevector <2 x float> [[TMP13]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT: [[R3:%.*]] = shufflevector <8 x float> [[R2]], <8 x float> [[TMP12]], <8 x i32> <i32 0, i32 1, i32 2, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
; SSE-NEXT: [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; SSE-NEXT: [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
; SSE-NEXT: [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
@@ -73,20 +73,18 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
; AVX-NEXT: ret <8 x float> [[R71]]
;
; AVX2-LABEL: @ceil_floor(
-; AVX2-NEXT: [[A0:%.*]] = extractelement <8 x float> [[TMP1:%.*]], i32 0
-; AVX2-NEXT: [[A3:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
-; AVX2-NEXT: [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; AVX2-NEXT: [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
+; AVX2-NEXT: [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
; AVX2-NEXT: [[TMP10:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP2]])
-; AVX2-NEXT: [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
+; AVX2-NEXT: [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 0, i32 3>
+; AVX2-NEXT: [[TMP12:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP11]])
; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
; AVX2-NEXT: [[TMP4:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
; AVX2-NEXT: [[TMP5:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
; AVX2-NEXT: [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
-; AVX2-NEXT: [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i32 0
+; AVX2-NEXT: [[R0:%.*]] = shufflevector <2 x float> [[TMP12]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; AVX2-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; AVX2-NEXT: [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT: [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i32 3
+; AVX2-NEXT: [[R3:%.*]] = shufflevector <8 x float> [[R23]], <8 x float> [[R0]], <8 x i32> <i32 0, i32 1, i32 2, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
; AVX2-NEXT: [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; AVX2-NEXT: [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
; AVX2-NEXT: [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
index 570d5a2500c96..e44a93b3e3aad 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls.ll
@@ -8,20 +8,21 @@
define <8 x float> @ceil_floor(<8 x float> %a) {
; SSE-LABEL: @ceil_floor(
-; SSE-NEXT: [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i32 0
-; SSE-NEXT: [[A3:%.*]] = extractelement <8 x float> [[A]], i32 3
-; SSE-NEXT: [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; SSE-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
+; SSE-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
; SSE-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP1]])
-; SSE-NEXT: [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
+; SSE-NEXT: [[TMP10:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 0, i32 3>
+; SSE-NEXT: [[TMP11:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP10]])
; SSE-NEXT: [[TMP3:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
; SSE-NEXT: [[TMP4:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
; SSE-NEXT: [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
; SSE-NEXT: [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
-; SSE-NEXT: [[R0:%.*]] = insertelement <8 x float> zeroinitializer, float [[AB0]], i32 0
+; SSE-NEXT: [[TMP14:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT: [[R0:%.*]] = shufflevector <8 x float> zeroinitializer, <8 x float> [[TMP14]], <8 x i32> <i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; SSE-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; SSE-NEXT: [[R2:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 3, i32 4, i32 5, i32 6, i32 7>
-; SSE-NEXT: [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i32 3
+; SSE-NEXT: [[TMP12:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <8 x i32> <i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT: [[TMP13:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT: [[R3:%.*]] = shufflevector <8 x float> [[R2]], <8 x float> [[TMP13]], <8 x i32> <i32 0, i32 1, i32 2, i32 9, i32 4, i32 5, i32 6, i32 7>
; SSE-NEXT: [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; SSE-NEXT: [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
; SSE-NEXT: [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
@@ -73,20 +74,19 @@ define <8 x float> @ceil_floor(<8 x float> %a) {
; AVX-NEXT: ret <8 x float> [[R71]]
;
; AVX2-LABEL: @ceil_floor(
-; AVX2-NEXT: [[A0:%.*]] = extractelement <8 x float> [[TMP1:%.*]], i32 0
-; AVX2-NEXT: [[A3:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
-; AVX2-NEXT: [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
-; AVX2-NEXT: [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
+; AVX2-NEXT: [[TMP2:%.*]] = shufflevector <8 x float> [[TMP1:%.*]], <8 x float> poison, <2 x i32> <i32 1, i32 2>
; AVX2-NEXT: [[TMP10:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP2]])
-; AVX2-NEXT: [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
+; AVX2-NEXT: [[TMP11:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 0, i32 3>
+; AVX2-NEXT: [[TMP12:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP11]])
; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 4, i32 5>
; AVX2-NEXT: [[TMP4:%.*]] = call <2 x float> @llvm.ceil.v2f32(<2 x float> [[TMP3]])
; AVX2-NEXT: [[TMP5:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <2 x i32> <i32 6, i32 7>
; AVX2-NEXT: [[TMP6:%.*]] = call <2 x float> @llvm.floor.v2f32(<2 x float> [[TMP5]])
-; AVX2-NEXT: [[R0:%.*]] = insertelement <8 x float> zeroinitializer, float [[AB0]], i32 0
+; AVX2-NEXT: [[TMP13:%.*]] = shufflevector <2 x float> [[TMP12]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[R0:%.*]] = shufflevector <8 x float> zeroinitializer, <8 x float> [[TMP13]], <8 x i32> <i32 8, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; AVX2-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; AVX2-NEXT: [[R23:%.*]] = shufflevector <8 x float> [[R0]], <8 x float> [[TMP7]], <8 x i32> <i32 0, i32 8, i32 9, i32 3, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT: [[R3:%.*]] = insertelement <8 x float> [[R23]], float [[AB3]], i32 3
+; AVX2-NEXT: [[R3:%.*]] = shufflevector <8 x float> [[R23]], <8 x float> [[TMP13]], <8 x i32> <i32 0, i32 1, i32 2, i32 9, i32 4, i32 5, i32 6, i32 7>
; AVX2-NEXT: [[TMP8:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; AVX2-NEXT: [[R52:%.*]] = shufflevector <8 x float> [[R3]], <8 x float> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
; AVX2-NEXT: [[TMP9:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll
index b126083d9b0f8..348264c3c2fc4 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll
@@ -492,28 +492,20 @@ define <8 x i32> @sdiv_v8i32_undefs(<8 x i32> %a) {
; AVX1-LABEL: @sdiv_v8i32_undefs(
; AVX1-NEXT: [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
; AVX1-NEXT: [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
-; AVX1-NEXT: [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
-; AVX1-NEXT: [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
; AVX1-NEXT: [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
-; AVX1-NEXT: [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
-; AVX1-NEXT: [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
; AVX1-NEXT: [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
; AVX1-NEXT: [[AB0:%.*]] = sdiv i32 [[A0]], undef
; AVX1-NEXT: [[AB1:%.*]] = sdiv i32 [[A1]], 4
-; AVX1-NEXT: [[AB2:%.*]] = sdiv i32 [[A2]], 8
-; AVX1-NEXT: [[AB3:%.*]] = sdiv i32 [[A3]], 16
; AVX1-NEXT: [[AB4:%.*]] = sdiv i32 [[A4]], undef
-; AVX1-NEXT: [[AB5:%.*]] = sdiv i32 [[A5]], 4
-; AVX1-NEXT: [[AB6:%.*]] = sdiv i32 [[A6]], 8
+; AVX1-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 2, i32 3, i32 5, i32 6>
+; AVX1-NEXT: [[TMP2:%.*]] = sdiv <4 x i32> [[TMP1]], <i32 8, i32 16, i32 4, i32 8>
; AVX1-NEXT: [[AB7:%.*]] = sdiv i32 [[A7]], 16
; AVX1-NEXT: [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
; AVX1-NEXT: [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
-; AVX1-NEXT: [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
-; AVX1-NEXT: [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; AVX1-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT: [[R3:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 poison, i32 10, i32 11, i32 poison>
; AVX1-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
-; AVX1-NEXT: [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
-; AVX1-NEXT: [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
-; AVX1-NEXT: [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; AVX1-NEXT: [[R7:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB7]], i32 7
; AVX1-NEXT: ret <8 x i32> [[R7]]
;
; AVX2-LABEL: @sdiv_v8i32_undefs(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
index 3dc47e7a362e5..ab28d14d27b8d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
@@ -492,28 +492,20 @@ define <8 x i32> @sdiv_v8i32_undefs(<8 x i32> %a) {
; AVX1-LABEL: @sdiv_v8i32_undefs(
; AVX1-NEXT: [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
; AVX1-NEXT: [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
-; AVX1-NEXT: [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
-; AVX1-NEXT: [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
; AVX1-NEXT: [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
-; AVX1-NEXT: [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
-; AVX1-NEXT: [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
; AVX1-NEXT: [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
; AVX1-NEXT: [[AB0:%.*]] = sdiv i32 [[A0]], undef
; AVX1-NEXT: [[AB1:%.*]] = sdiv i32 [[A1]], 4
-; AVX1-NEXT: [[AB2:%.*]] = sdiv i32 [[A2]], 8
-; AVX1-NEXT: [[AB3:%.*]] = sdiv i32 [[A3]], 16
; AVX1-NEXT: [[AB4:%.*]] = sdiv i32 [[A4]], undef
-; AVX1-NEXT: [[AB5:%.*]] = sdiv i32 [[A5]], 4
-; AVX1-NEXT: [[AB6:%.*]] = sdiv i32 [[A6]], 8
+; AVX1-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 2, i32 3, i32 5, i32 6>
+; AVX1-NEXT: [[TMP2:%.*]] = sdiv <4 x i32> [[TMP1]], <i32 8, i32 16, i32 4, i32 8>
; AVX1-NEXT: [[AB7:%.*]] = sdiv i32 [[A7]], 16
; AVX1-NEXT: [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
; AVX1-NEXT: [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
-; AVX1-NEXT: [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
-; AVX1-NEXT: [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; AVX1-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX1-NEXT: [[R3:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 10, i32 11, i32 7>
; AVX1-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
-; AVX1-NEXT: [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
-; AVX1-NEXT: [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
-; AVX1-NEXT: [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; AVX1-NEXT: [[R7:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB7]], i32 7
; AVX1-NEXT: ret <8 x i32> [[R7]]
;
; AVX2-LABEL: @sdiv_v8i32_undefs(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-fp-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp-inseltpoison.ll
index 0d853e9a3e401..1641141670ed8 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-fp-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp-inseltpoison.ll
@@ -607,34 +607,25 @@ define <8 x double> @buildvector_div_8f64(<8 x double> %a, <8 x double> %b) {
; SSE-NEXT: ret <8 x double> [[TMP1]]
;
; SLM-LABEL: @buildvector_div_8f64(
-; SLM-NEXT: [[A2:%.*]] = extractelement <8 x double> [[A:%.*]], i32 2
-; SLM-NEXT: [[A3:%.*]] = extractelement <8 x double> [[A]], i32 3
-; SLM-NEXT: [[A4:%.*]] = extractelement <8 x double> [[A]], i32 4
-; SLM-NEXT: [[A5:%.*]] = extractelement <8 x double> [[A]], i32 5
-; SLM-NEXT: [[A6:%.*]] = extractelement <8 x double> [[A]], i32 6
-; SLM-NEXT: [[A7:%.*]] = extractelement <8 x double> [[A]], i32 7
-; SLM-NEXT: [[B2:%.*]] = extractelement <8 x double> [[B:%.*]], i32 2
-; SLM-NEXT: [[B3:%.*]] = extractelement <8 x double> [[B]], i32 3
-; SLM-NEXT: [[B4:%.*]] = extractelement <8 x double> [[B]], i32 4
-; SLM-NEXT: [[B5:%.*]] = extractelement <8 x double> [[B]], i32 5
-; SLM-NEXT: [[B6:%.*]] = extractelement <8 x double> [[B]], i32 6
-; SLM-NEXT: [[B7:%.*]] = extractelement <8 x double> [[B]], i32 7
-; SLM-NEXT: [[TMP1:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <2 x i32> <i32 0, i32 1>
-; SLM-NEXT: [[TMP2:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> <i32 0, i32 1>
+; SLM-NEXT: [[TMP1:%.*]] = shufflevector <8 x double> [[A:%.*]], <8 x double> poison, <2 x i32> <i32 0, i32 1>
+; SLM-NEXT: [[TMP2:%.*]] = shufflevector <8 x double> [[B:%.*]], <8 x double> poison, <2 x i32> <i32 0, i32 1>
; SLM-NEXT: [[TMP12:%.*]] = fdiv <2 x double> [[TMP1]], [[TMP2]]
-; SLM-NEXT: [[C2:%.*]] = fdiv double [[A2]], [[B2]]
-; SLM-NEXT: [[C3:%.*]] = fdiv double [[A3]], [[B3]]
-; SLM-NEXT: [[C4:%.*]] = fdiv double [[A4]], [[B4]]
-; SLM-NEXT: [[C5:%.*]] = fdiv double [[A5]], [[B5]]
-; SLM-NEXT: [[C6:%.*]] = fdiv double [[A6]], [[B6]]
-; SLM-NEXT: [[C7:%.*]] = fdiv double [[A7]], [[B7]]
+; SLM-NEXT: [[TMP4:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <2 x i32> <i32 2, i32 3>
+; SLM-NEXT: [[TMP5:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> <i32 2, i32 3>
+; SLM-NEXT: [[TMP6:%.*]] = fdiv <2 x double> [[TMP4]], [[TMP5]]
+; SLM-NEXT: [[TMP7:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <2 x i32> <i32 4, i32 5>
+; SLM-NEXT: [[TMP8:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> <i32 4, i32 5>
+; SLM-NEXT: [[TMP9:%.*]] = fdiv <2 x double> [[TMP7]], [[TMP8]]
+; SLM-NEXT: [[TMP10:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <2 x i32> <i32 6, i32 7>
+; SLM-NEXT: [[TMP11:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> <i32 6, i32 7>
+; SLM-NEXT: [[TMP13:%.*]] = fdiv <2 x double> [[TMP10]], [[TMP11]]
; SLM-NEXT: [[TMP16:%.*]] = shufflevector <2 x double> [[TMP12]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SLM-NEXT: [[R2:%.*]] = insertelement <8 x double> [[TMP16]], double [[C2]], i32 2
-; SLM-NEXT: [[R3:%.*]] = insertelement <8 x double> [[R2]], double [[C3]], i32 3
-; SLM-NEXT: [[R4:%.*]] = insertelement <8 x double> [[R3]], double [[C4]], i32 4
-; SLM-NEXT: [[R5:%.*]] = insertelement <8 x double> [[R4]], double [[C5]], i32 5
-; SLM-NEXT: [[R6:%.*]] = insertelement <8 x double> [[R5]], double [[C6]], i32 6
-; SLM-NEXT: [[R73:%.*]] = insertelement <8 x double> [[R6]], double [[C7]], i32 7
+; SLM-NEXT: [[TMP14:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT: [[TMP15:%.*]] = shufflevector <2 x double> [[TMP12]], <2 x double> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT: [[TMP19:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT: [[TMP17:%.*]] = shufflevector <8 x double> [[TMP15]], <8 x double> [[TMP19]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
+; SLM-NEXT: [[TMP18:%.*]] = shufflevector <2 x double> [[TMP13]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT: [[R73:%.*]] = shufflevector <8 x double> [[TMP17]], <8 x double> [[TMP18]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
; SLM-NEXT: ret <8 x double> [[R73]]
;
; AVX-LABEL: @buildvector_div_8f64(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll
index 633ed4e239e16..2f8332d0b4182 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll
@@ -607,34 +607,25 @@ define <8 x double> @buildvector_div_8f64(<8 x double> %a, <8 x double> %b) {
; SSE-NEXT: ret <8 x double> [[TMP1]]
;
; SLM-LABEL: @buildvector_div_8f64(
-; SLM-NEXT: [[A2:%.*]] = extractelement <8 x double> [[A:%.*]], i32 2
-; SLM-NEXT: [[A3:%.*]] = extractelement <8 x double> [[A]], i32 3
-; SLM-NEXT: [[A4:%.*]] = extractelement <8 x double> [[A]], i32 4
-; SLM-NEXT: [[A5:%.*]] = extractelement <8 x double> [[A]], i32 5
-; SLM-NEXT: [[A6:%.*]] = extractelement <8 x double> [[A]], i32 6
-; SLM-NEXT: [[A7:%.*]] = extractelement <8 x double> [[A]], i32 7
-; SLM-NEXT: [[B2:%.*]] = extractelement <8 x double> [[B:%.*]], i32 2
-; SLM-NEXT: [[B3:%.*]] = extractelement <8 x double> [[B]], i32 3
-; SLM-NEXT: [[B4:%.*]] = extractelement <8 x double> [[B]], i32 4
-; SLM-NEXT: [[B5:%.*]] = extractelement <8 x double> [[B]], i32 5
-; SLM-NEXT: [[B6:%.*]] = extractelement <8 x double> [[B]], i32 6
-; SLM-NEXT: [[B7:%.*]] = extractelement <8 x double> [[B]], i32 7
-; SLM-NEXT: [[TMP1:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <2 x i32> <i32 0, i32 1>
-; SLM-NEXT: [[TMP2:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> <i32 0, i32 1>
+; SLM-NEXT: [[TMP1:%.*]] = shufflevector <8 x double> [[A:%.*]], <8 x double> poison, <2 x i32> <i32 0, i32 1>
+; SLM-NEXT: [[TMP2:%.*]] = shufflevector <8 x double> [[B:%.*]], <8 x double> poison, <2 x i32> <i32 0, i32 1>
; SLM-NEXT: [[TMP12:%.*]] = fdiv <2 x double> [[TMP1]], [[TMP2]]
-; SLM-NEXT: [[C2:%.*]] = fdiv double [[A2]], [[B2]]
-; SLM-NEXT: [[C3:%.*]] = fdiv double [[A3]], [[B3]]
-; SLM-NEXT: [[C4:%.*]] = fdiv double [[A4]], [[B4]]
-; SLM-NEXT: [[C5:%.*]] = fdiv double [[A5]], [[B5]]
-; SLM-NEXT: [[C6:%.*]] = fdiv double [[A6]], [[B6]]
-; SLM-NEXT: [[C7:%.*]] = fdiv double [[A7]], [[B7]]
+; SLM-NEXT: [[TMP4:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <2 x i32> <i32 2, i32 3>
+; SLM-NEXT: [[TMP5:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> <i32 2, i32 3>
+; SLM-NEXT: [[TMP6:%.*]] = fdiv <2 x double> [[TMP4]], [[TMP5]]
+; SLM-NEXT: [[TMP7:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <2 x i32> <i32 4, i32 5>
+; SLM-NEXT: [[TMP8:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> <i32 4, i32 5>
+; SLM-NEXT: [[TMP9:%.*]] = fdiv <2 x double> [[TMP7]], [[TMP8]]
+; SLM-NEXT: [[TMP10:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <2 x i32> <i32 6, i32 7>
+; SLM-NEXT: [[TMP11:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> <i32 6, i32 7>
+; SLM-NEXT: [[TMP13:%.*]] = fdiv <2 x double> [[TMP10]], [[TMP11]]
; SLM-NEXT: [[TMP16:%.*]] = shufflevector <2 x double> [[TMP12]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SLM-NEXT: [[R2:%.*]] = insertelement <8 x double> [[TMP16]], double [[C2]], i32 2
-; SLM-NEXT: [[R3:%.*]] = insertelement <8 x double> [[R2]], double [[C3]], i32 3
-; SLM-NEXT: [[R4:%.*]] = insertelement <8 x double> [[R3]], double [[C4]], i32 4
-; SLM-NEXT: [[R5:%.*]] = insertelement <8 x double> [[R4]], double [[C5]], i32 5
-; SLM-NEXT: [[R6:%.*]] = insertelement <8 x double> [[R5]], double [[C6]], i32 6
-; SLM-NEXT: [[R73:%.*]] = insertelement <8 x double> [[R6]], double [[C7]], i32 7
+; SLM-NEXT: [[TMP14:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT: [[TMP15:%.*]] = shufflevector <2 x double> [[TMP12]], <2 x double> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT: [[TMP19:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT: [[TMP17:%.*]] = shufflevector <8 x double> [[TMP15]], <8 x double> [[TMP19]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
+; SLM-NEXT: [[TMP18:%.*]] = shufflevector <2 x double> [[TMP13]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SLM-NEXT: [[R73:%.*]] = shufflevector <8 x double> [[TMP17]], <8 x double> [[TMP18]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
; SLM-NEXT: ret <8 x double> [[R73]]
;
; AVX-LABEL: @buildvector_div_8f64(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/broadcast.ll b/llvm/test/Transforms/SLPVectorizer/X86/broadcast.ll
index 0e3d51b88e4aa..c16f8d8948320 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/broadcast.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/broadcast.ll
@@ -15,11 +15,12 @@ define void @bcast_vals(ptr %A, ptr %B, ptr %S) {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[A0:%.*]] = load i64, ptr [[A:%.*]], align 8
; CHECK-NEXT: [[B0:%.*]] = load i64, ptr [[B:%.*]], align 8
-; CHECK-NEXT: [[V1:%.*]] = sub i64 [[A0]], 1
-; CHECK-NEXT: [[V2:%.*]] = sub i64 [[B0]], 1
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i64> poison, i64 [[V1]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> poison, i64 [[A0]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i64> [[TMP5]], i64 [[B0]], i64 1
+; CHECK-NEXT: [[TMP7:%.*]] = sub <2 x i64> [[TMP6]], splat (i64 1)
+; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <2 x i64> [[TMP7]], <2 x i64> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[TMP0]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> poison, i64 [[V2]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i64> [[TMP7]], <2 x i64> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[TMP1]], [[TMP3]]
; CHECK-NEXT: store <4 x i64> [[TMP4]], ptr [[S:%.*]], align 8
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/c-ray.ll b/llvm/test/Transforms/SLPVectorizer/X86/c-ray.ll
index f80db40aef3a4..79945d8372b05 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/c-ray.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/c-ray.ll
@@ -12,32 +12,34 @@ define i32 @ray_sphere(ptr nocapture noundef readonly %sph, ptr nocapture nounde
; SSE2-LABEL: @ray_sphere(
; SSE2-NEXT: entry:
; SSE2-NEXT: [[DIR:%.*]] = getelementptr inbounds [[STRUCT_RAY:%.*]], ptr [[RAY:%.*]], i64 0, i32 1
-; SSE2-NEXT: [[TMP0:%.*]] = load double, ptr [[DIR]], align 8
-; SSE2-NEXT: [[Y:%.*]] = getelementptr inbounds [[STRUCT_RAY]], ptr [[RAY]], i64 0, i32 1, i32 1
-; SSE2-NEXT: [[TMP1:%.*]] = load double, ptr [[Y]], align 8
-; SSE2-NEXT: [[MUL6:%.*]] = fmul double [[TMP1]], [[TMP1]]
-; SSE2-NEXT: [[TMP2:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP0]], double [[TMP0]], double [[MUL6]])
; SSE2-NEXT: [[Z:%.*]] = getelementptr inbounds [[STRUCT_RAY]], ptr [[RAY]], i64 0, i32 1, i32 2
; SSE2-NEXT: [[TMP3:%.*]] = load double, ptr [[Z]], align 8
-; SSE2-NEXT: [[TMP4:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP3]], double [[TMP3]], double [[TMP2]])
-; SSE2-NEXT: [[MUL:%.*]] = fmul double [[TMP0]], 2.000000e+00
; SSE2-NEXT: [[TMP5:%.*]] = load double, ptr [[RAY]], align 8
; SSE2-NEXT: [[TMP6:%.*]] = load double, ptr [[SPH:%.*]], align 8
; SSE2-NEXT: [[SUB:%.*]] = fsub double [[TMP5]], [[TMP6]]
-; SSE2-NEXT: [[MUL17:%.*]] = fmul double [[TMP1]], 2.000000e+00
; SSE2-NEXT: [[Y19:%.*]] = getelementptr inbounds [[STRUCT_VEC3:%.*]], ptr [[RAY]], i64 0, i32 1
; SSE2-NEXT: [[TMP7:%.*]] = load double, ptr [[Y19]], align 8
; SSE2-NEXT: [[Y21:%.*]] = getelementptr inbounds [[STRUCT_VEC3]], ptr [[SPH]], i64 0, i32 1
; SSE2-NEXT: [[TMP8:%.*]] = load double, ptr [[Y21]], align 8
; SSE2-NEXT: [[SUB22:%.*]] = fsub double [[TMP7]], [[TMP8]]
-; SSE2-NEXT: [[MUL23:%.*]] = fmul double [[MUL17]], [[SUB22]]
-; SSE2-NEXT: [[TMP9:%.*]] = tail call double @llvm.fmuladd.f64(double [[MUL]], double [[SUB]], double [[MUL23]])
+; SSE2-NEXT: [[TMP39:%.*]] = load <2 x double>, ptr [[DIR]], align 8
+; SSE2-NEXT: [[TMP40:%.*]] = shufflevector <2 x double> [[TMP39]], <2 x double> poison, <2 x i32> zeroinitializer
+; SSE2-NEXT: [[TMP41:%.*]] = fmul <2 x double> [[TMP40]], <double 1.000000e+00, double 2.000000e+00>
+; SSE2-NEXT: [[TMP42:%.*]] = shufflevector <2 x double> [[TMP39]], <2 x double> poison, <2 x i32> <i32 1, i32 1>
+; SSE2-NEXT: [[TMP43:%.*]] = fmul <2 x double> [[TMP42]], <double 1.000000e+00, double 2.000000e+00>
+; SSE2-NEXT: [[TMP44:%.*]] = insertelement <2 x double> [[TMP42]], double [[SUB22]], i64 1
+; SSE2-NEXT: [[TMP45:%.*]] = fmul <2 x double> [[TMP43]], [[TMP44]]
+; SSE2-NEXT: [[TMP46:%.*]] = insertelement <2 x double> [[TMP40]], double [[SUB]], i64 1
+; SSE2-NEXT: [[TMP47:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP41]], <2 x double> [[TMP46]], <2 x double> [[TMP45]])
+; SSE2-NEXT: [[TMP48:%.*]] = extractelement <2 x double> [[TMP47]], i64 0
+; SSE2-NEXT: [[TMP4:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP3]], double [[TMP3]], double [[TMP48]])
; SSE2-NEXT: [[MUL26:%.*]] = fmul double [[TMP3]], 2.000000e+00
; SSE2-NEXT: [[Z28:%.*]] = getelementptr inbounds [[STRUCT_VEC3]], ptr [[RAY]], i64 0, i32 2
; SSE2-NEXT: [[TMP10:%.*]] = load double, ptr [[Z28]], align 8
; SSE2-NEXT: [[Z30:%.*]] = getelementptr inbounds [[STRUCT_VEC3]], ptr [[SPH]], i64 0, i32 2
; SSE2-NEXT: [[TMP11:%.*]] = load double, ptr [[Z30]], align 8
; SSE2-NEXT: [[SUB31:%.*]] = fsub double [[TMP10]], [[TMP11]]
+; SSE2-NEXT: [[TMP9:%.*]] = extractelement <2 x double> [[TMP47]], i64 1
; SSE2-NEXT: [[TMP12:%.*]] = tail call double @llvm.fmuladd.f64(double [[MUL26]], double [[SUB31]], double [[TMP9]])
; SSE2-NEXT: [[MUL42:%.*]] = fmul double [[TMP8]], [[TMP8]]
; SSE2-NEXT: [[TMP13:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP6]], double [[TMP6]], double [[MUL42]])
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/copyable-operands-reordering.ll b/llvm/test/Transforms/SLPVectorizer/X86/copyable-operands-reordering.ll
index 52399974d31c6..bfb9f92528233 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/copyable-operands-reordering.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/copyable-operands-reordering.ll
@@ -7,19 +7,17 @@ define i64 @test(ptr %buf) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = load i8, ptr [[BUF]], align 1
; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[BUF]], i64 1
-; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
-; CHECK-NEXT: [[ARRAYIDX22:%.*]] = getelementptr inbounds nuw i8, ptr [[BUF]], i64 2
-; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[ARRAYIDX22]], align 1
; CHECK-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds nuw i8, ptr [[BUF]], i64 3
; CHECK-NEXT: [[TMP3:%.*]] = zext i8 [[TMP0]] to i32
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> <i32 poison, i32 1, i32 1, i32 1>, i32 [[TMP3]], i64 0
; CHECK-NEXT: [[TMP6:%.*]] = mul <4 x i32> <i32 24, i32 0, i32 0, i32 0>, [[TMP5]]
-; CHECK-NEXT: [[TMP17:%.*]] = zext i8 [[TMP8]] to i32
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i32> <i32 poison, i32 1, i32 1, i32 1>, i32 [[TMP17]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = load <2 x i8>, ptr [[ARRAYIDX1]], align 1
+; CHECK-NEXT: [[TMP10:%.*]] = zext <2 x i8> [[TMP8]] to <2 x i32>
+; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <2 x i32> [[TMP10]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <4 x i32> <i32 poison, i32 1, i32 1, i32 1>, <4 x i32> [[TMP17]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[TMP4:%.*]] = mul <4 x i32> <i32 16, i32 0, i32 0, i32 0>, [[TMP20]]
; CHECK-NEXT: [[TMP7:%.*]] = or disjoint <4 x i32> [[TMP4]], [[TMP6]]
-; CHECK-NEXT: [[TMP21:%.*]] = zext i8 [[TMP10]] to i32
-; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i32> <i32 poison, i32 1, i32 1, i32 1>, i32 [[TMP21]], i64 0
+; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x i32> <i32 poison, i32 1, i32 1, i32 1>, <4 x i32> [[TMP17]], <4 x i32> <i32 5, i32 1, i32 2, i32 3>
; CHECK-NEXT: [[TMP9:%.*]] = mul <4 x i32> <i32 8, i32 0, i32 0, i32 0>, [[TMP22]]
; CHECK-NEXT: [[TMP18:%.*]] = or disjoint <4 x i32> [[TMP7]], [[TMP9]]
; CHECK-NEXT: [[TMP19:%.*]] = load <4 x i8>, ptr [[ARRAYIDX8]], align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_getpointersdiff-nullopt.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_getpointersdiff-nullopt.ll
index cbbac32febdb3..b7af61c6316c8 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/crash_getpointersdiff-nullopt.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_getpointersdiff-nullopt.ll
@@ -26,18 +26,18 @@ define void @test(i64 %arg0, i64 %arg1) {
; CHECK-NEXT: [[IDX4_SCALED:%.*]] = shl i64 [[IDX4]], 3
; CHECK-NEXT: [[GEP4:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[IDX4_SCALED]]
; CHECK-NEXT: [[GEP4_OFF:%.*]] = getelementptr i8, ptr [[GEP4]], i64 -8
-; CHECK-NEXT: [[LOAD4:%.*]] = load double, ptr [[GEP4_OFF]], align 8
-; CHECK-NEXT: [[LOAD5:%.*]] = load double, ptr [[GEP4]], align 8
+; CHECK-NEXT: [[TMP7:%.*]] = load <2 x double>, ptr [[GEP4_OFF]], align 8
; CHECK-NEXT: br label [[REDUCE]]
; CHECK: dead:
; CHECK-NEXT: br label [[REDUCE]]
; CHECK: reduce:
-; CHECK-NEXT: [[PHI4:%.*]] = phi double [ [[LOAD4]], [[LOOP]] ], [ 0.000000e+00, [[DEAD:%.*]] ]
-; CHECK-NEXT: [[PHI5:%.*]] = phi double [ [[LOAD5]], [[LOOP]] ], [ 0.000000e+00, [[DEAD]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x double> [ [[TMP0]], [[LOOP]] ], [ poison, [[DEAD]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x double> [ [[TMP0]], [[LOOP]] ], [ poison, [[DEAD:%.*]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = phi <2 x double> [ [[TMP7]], [[LOOP]] ], [ poison, [[DEAD]] ]
; CHECK-NEXT: [[TMP2:%.*]] = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> [[TMP1]])
-; CHECK-NEXT: [[TMP3:%.*]] = call double @llvm.minimum.f64(double [[TMP2]], double [[PHI4]])
-; CHECK-NEXT: [[TMP4:%.*]] = call double @llvm.minimum.f64(double [[PHI5]], double 0.000000e+00)
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x double> <double poison, double 0.000000e+00>, double [[TMP2]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = call <2 x double> @llvm.minimum.v2f64(<2 x double> [[TMP9]], <2 x double> [[TMP8]])
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP6]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP6]], i64 1
; CHECK-NEXT: [[TMP5:%.*]] = call double @llvm.minimum.f64(double [[TMP3]], double [[TMP4]])
; CHECK-NEXT: [[MIN6:%.*]] = call double @llvm.minimum.f64(double [[TMP5]], double 0.000000e+00)
; CHECK-NEXT: [[COUNTER_NEXT]] = add i64 [[COUNTER]], 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/debug-info-salvage.ll b/llvm/test/Transforms/SLPVectorizer/X86/debug-info-salvage.ll
index ae721cb84750f..613d9bfc0fc5f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/debug-info-salvage.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/debug-info-salvage.ll
@@ -19,9 +19,6 @@ define void @test(i8 %a, i8 %b, ptr %p) {
; CHECK-NEXT: [[BLOCK_COLOR_SROA_7_0_INSERT_SHIFT_I:%.*]] = shl i32 [[BLOCK_COLOR_SROA_7_0_INSERT_EXT_I]], 16
; CHECK-NEXT: [[BLOCK_COLOR_SROA_5_0_INSERT_EXT_I:%.*]] = select i1 false, i32 0, i32 [[TMP0]]
; CHECK-NEXT: [[BLOCK_COLOR_SROA_5_0_INSERT_SHIFT_I:%.*]] = shl i32 [[BLOCK_COLOR_SROA_5_0_INSERT_EXT_I]], 0
-; CHECK-NEXT: [[OP_RDX:%.*]] = or i32 0, [[BLOCK_COLOR_SROA_7_0_INSERT_SHIFT_I]]
-; CHECK-NEXT: [[OP_RDX1:%.*]] = or i32 [[OP_RDX]], [[BLOCK_COLOR_SROA_5_0_INSERT_SHIFT_I]]
-; CHECK-NEXT: store i32 [[OP_RDX1]], ptr null, align 4
; CHECK-NEXT: [[ADD46_1_I:%.*]] = or i32 0, [[RETVAL_SROA_2_0_INSERT_EXT_I_I]]
; CHECK-NEXT: [[ADD49_1_I:%.*]] = or i32 0, [[RETVAL_SROA_3_0_INSERT_EXT_I_I]]
; CHECK-NEXT: [[CMP_I11_I_I_1_I:%.*]] = icmp slt i32 [[ADD46_1_I]], 0
@@ -30,10 +27,14 @@ define void @test(i8 %a, i8 %b, ptr %p) {
; CHECK-NEXT: [[TMP17:%.*]] = insertelement <2 x i1> <i1 false, i1 poison>, i1 [[CMP_I11_I_I_1_I]], i64 1
; CHECK-NEXT: [[TMP19:%.*]] = select <2 x i1> [[TMP17]], <2 x i32> zeroinitializer, <2 x i32> [[TMP18]]
; CHECK-NEXT: [[TMP20:%.*]] = shl <2 x i32> [[TMP19]], <i32 16, i32 0>
-; CHECK-NEXT: [[TMP21:%.*]] = extractelement <2 x i32> [[TMP20]], i64 0
-; CHECK-NEXT: [[OP_RDX2:%.*]] = or i32 0, [[TMP21]]
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x i32> [[TMP20]], i64 1
-; CHECK-NEXT: [[OP_RDX3:%.*]] = or i32 [[OP_RDX2]], [[TMP22]]
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP20]], <2 x i32> poison, <2 x i32> <i32 poison, i32 0>
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> [[TMP6]], i32 [[BLOCK_COLOR_SROA_7_0_INSERT_SHIFT_I]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = or <2 x i32> zeroinitializer, [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x i32> [[TMP20]], i32 [[BLOCK_COLOR_SROA_5_0_INSERT_SHIFT_I]], i64 0
+; CHECK-NEXT: [[TMP10:%.*]] = or <2 x i32> [[TMP8]], [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[TMP10]], i64 0
+; CHECK-NEXT: store i32 [[TMP11]], ptr null, align 4
+; CHECK-NEXT: [[OP_RDX3:%.*]] = extractelement <2 x i32> [[TMP10]], i64 1
; CHECK-NEXT: store i32 [[OP_RDX3]], ptr [[ARRAYIDX51_1_I]], align 4
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/deleted-inst-reduction-attempt.ll b/llvm/test/Transforms/SLPVectorizer/X86/deleted-inst-reduction-attempt.ll
index 4d8233844404c..efbdd2fc06ba3 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/deleted-inst-reduction-attempt.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/deleted-inst-reduction-attempt.ll
@@ -22,13 +22,12 @@ define double @test(ptr %0, double %1) {
; CHECK-NEXT: [[TMP19:%.*]] = fadd <4 x double> [[TMP16]], [[TMP18]]
; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <4 x double> [[TMP15]], <4 x double> <double 0.000000e+00, double poison, double 0.000000e+00, double 1.000000e+00>, <4 x i32> <i32 4, i32 3, i32 6, i32 7>
; CHECK-NEXT: [[TMP21:%.*]] = fmul <4 x double> [[TMP20]], [[TMP19]]
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x double> [[TMP21]], i64 0
-; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x double> [[TMP16]], i64 0
-; CHECK-NEXT: [[TMP24:%.*]] = fadd double [[TMP23]], [[TMP22]]
-; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x double> [[TMP21]], i64 1
-; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x double> [[TMP21]], i64 2
-; CHECK-NEXT: [[TMP27:%.*]] = fadd double [[TMP25]], [[TMP26]]
+; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <4 x double> [[TMP21]], <4 x double> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x double> [[TMP16]], <4 x double> [[TMP21]], <2 x i32> <i32 0, i32 6>
+; CHECK-NEXT: [[TMP23:%.*]] = fadd <2 x double> [[TMP25]], [[TMP22]]
+; CHECK-NEXT: [[TMP27:%.*]] = extractelement <2 x double> [[TMP23]], i64 1
; CHECK-NEXT: [[TMP28:%.*]] = fadd double [[TMP27]], [[TMP6]]
+; CHECK-NEXT: [[TMP24:%.*]] = extractelement <2 x double> [[TMP23]], i64 0
; CHECK-NEXT: [[TMP29:%.*]] = fadd double [[TMP24]], [[TMP28]]
; CHECK-NEXT: ret double [[TMP29]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extractelement-multi-register-use.ll b/llvm/test/Transforms/SLPVectorizer/X86/extractelement-multi-register-use.ll
index 3a66f79121163..b59655404f889 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extractelement-multi-register-use.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extractelement-multi-register-use.ll
@@ -8,14 +8,15 @@ define void @test(double %i) {
; CHECK-NEXT: [[I74:%.*]] = fsub double 0.000000e+00, poison
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> <double 0.000000e+00, double poison>, double [[I]], i64 1
; CHECK-NEXT: [[TMP3:%.*]] = fsub <2 x double> zeroinitializer, [[TMP2]]
-; CHECK-NEXT: [[I96:%.*]] = fsub double poison, 0.000000e+00
-; CHECK-NEXT: [[I75:%.*]] = fsub double 0.000000e+00, [[I]]
+; CHECK-NEXT: [[TMP17:%.*]] = fsub <2 x double> <double poison, double 0.000000e+00>, [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x double> <double poison, double 0.000000e+00>, double [[I]], i64 0
; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> <double 0.000000e+00, double poison>, <2 x i32> <i32 2, i32 0>
; CHECK-NEXT: [[TMP15:%.*]] = fsub <2 x double> [[TMP4]], [[TMP8]]
; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x double> [[TMP15]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <8 x double> <double 0.000000e+00, double 0.000000e+00, double poison, double poison, double 0.000000e+00, double poison, double poison, double poison>, <8 x double> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <8 x double> [[TMP6]], double [[I75]], i64 5
+; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <2 x double> [[TMP17]], <2 x double> poison, <8 x i32> <i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP17]], <2 x double> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <8 x double> [[TMP6]], <8 x double> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 9, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <8 x double> [[TMP7]], <8 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 2, i32 3>
; CHECK-NEXT: [[TMP11:%.*]] = fmul <8 x double> zeroinitializer, [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = fadd <8 x double> zeroinitializer, [[TMP11]]
@@ -23,14 +24,12 @@ define void @test(double %i) {
; CHECK-NEXT: [[TMP14:%.*]] = fcmp ult <8 x double> [[TMP13]], zeroinitializer
; CHECK-NEXT: br label [[BB116:%.*]]
; CHECK: bb116:
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x double> [[TMP15]], i64 0
-; CHECK-NEXT: [[I117:%.*]] = fmul double 0.000000e+00, [[TMP16]]
-; CHECK-NEXT: [[I119:%.*]] = fmul double 0.000000e+00, [[I96]]
-; CHECK-NEXT: [[I120:%.*]] = fadd double [[I117]], [[I119]]
-; CHECK-NEXT: [[TMP21:%.*]] = fmul double 0.000000e+00, [[I74]]
+; CHECK-NEXT: [[TMP16:%.*]] = fmul <2 x double> zeroinitializer, [[TMP17]]
; CHECK-NEXT: [[TMP19:%.*]] = fmul <2 x double> zeroinitializer, [[TMP3]]
-; CHECK-NEXT: [[TMP20:%.*]] = fmul double 0.000000e+00, [[I75]]
-; CHECK-NEXT: [[I128:%.*]] = fadd double [[TMP20]], [[TMP21]]
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <2 x double> [[TMP15]], double [[I74]], i64 1
+; CHECK-NEXT: [[TMP28:%.*]] = fmul <2 x double> zeroinitializer, [[TMP18]]
+; CHECK-NEXT: [[TMP20:%.*]] = fadd <2 x double> [[TMP16]], [[TMP28]]
+; CHECK-NEXT: [[I128:%.*]] = extractelement <2 x double> [[TMP20]], i64 1
; CHECK-NEXT: [[I139:%.*]] = call double @llvm.maxnum.f64(double [[I128]], double 0.000000e+00)
; CHECK-NEXT: [[TMP22:%.*]] = fadd <2 x double> [[TMP19]], zeroinitializer
; CHECK-NEXT: [[TMP23:%.*]] = call <2 x double> @llvm.maxnum.v2f64(<2 x double> [[TMP22]], <2 x double> zeroinitializer)
@@ -38,6 +37,7 @@ define void @test(double %i) {
; CHECK-NEXT: [[TMP25:%.*]] = fptosi <2 x double> [[TMP24]] to <2 x i32>
; CHECK-NEXT: [[TMP26:%.*]] = sub <2 x i32> zeroinitializer, [[TMP25]]
; CHECK-NEXT: [[TMP27:%.*]] = icmp sgt <2 x i32> [[TMP26]], zeroinitializer
+; CHECK-NEXT: [[I120:%.*]] = extractelement <2 x double> [[TMP20]], i64 0
; CHECK-NEXT: [[I147:%.*]] = fcmp ogt double [[I120]], 0.000000e+00
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
index 2c0787e9bc65c..e7e7e9aa1697d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
@@ -87,15 +87,17 @@ define float @bazz() {
; THRESHOLD-LABEL: @bazz(
; THRESHOLD-NEXT: entry:
; THRESHOLD-NEXT: [[TMP0:%.*]] = load i32, ptr @n, align 4
-; THRESHOLD-NEXT: [[MUL:%.*]] = mul nsw i32 [[TMP0]], 3
-; THRESHOLD-NEXT: [[CONV:%.*]] = sitofp i32 [[MUL]] to float
-; THRESHOLD-NEXT: [[MUL5:%.*]] = shl nsw i32 [[TMP0]], 2
-; THRESHOLD-NEXT: [[CONV6:%.*]] = sitofp i32 [[MUL5]] to float
+; THRESHOLD-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
+; THRESHOLD-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> poison, <2 x i32> zeroinitializer
+; THRESHOLD-NEXT: [[TMP7:%.*]] = mul nsw <2 x i32> [[TMP6]], <i32 3, i32 4>
+; THRESHOLD-NEXT: [[TMP8:%.*]] = sitofp <2 x i32> [[TMP7]] to <2 x float>
; THRESHOLD-NEXT: [[TMP1:%.*]] = load <8 x float>, ptr @arr, align 16
; THRESHOLD-NEXT: [[TMP2:%.*]] = load <8 x float>, ptr @arr1, align 16
; THRESHOLD-NEXT: [[TMP3:%.*]] = fmul fast <8 x float> [[TMP2]], [[TMP1]]
; THRESHOLD-NEXT: [[TMP4:%.*]] = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> [[TMP3]])
+; THRESHOLD-NEXT: [[CONV:%.*]] = extractelement <2 x float> [[TMP8]], i64 0
; THRESHOLD-NEXT: [[OP_RDX:%.*]] = fadd fast float [[TMP4]], [[CONV]]
+; THRESHOLD-NEXT: [[CONV6:%.*]] = extractelement <2 x float> [[TMP8]], i64 1
; THRESHOLD-NEXT: [[OP_RDX1:%.*]] = fadd fast float [[OP_RDX]], [[CONV6]]
; THRESHOLD-NEXT: store float [[OP_RDX1]], ptr @res, align 4
; THRESHOLD-NEXT: ret float [[OP_RDX1]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/intrinsic.ll b/llvm/test/Transforms/SLPVectorizer/X86/intrinsic.ll
index 47c34fa3ff19f..e84b068a2cdd6 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/intrinsic.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/intrinsic.ll
@@ -180,28 +180,17 @@ entry:
define void @vec_ctlz_i32_neg(ptr %a, ptr %b, ptr %c, i1) {
; CHECK-LABEL: @vec_ctlz_i32_neg(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[I0:%.*]] = load i32, ptr [[A:%.*]], align 4
-; CHECK-NEXT: [[I1:%.*]] = load i32, ptr [[B:%.*]], align 4
-; CHECK-NEXT: [[ADD1:%.*]] = add i32 [[I0]], [[I1]]
-; CHECK-NEXT: [[CALL1:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[ADD1]], i1 true) #[[ATTR5:[0-9]+]]
-; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 1
-; CHECK-NEXT: [[I2:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
-; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 1
-; CHECK-NEXT: [[I3:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4
-; CHECK-NEXT: [[ADD2:%.*]] = add i32 [[I2]], [[I3]]
-; CHECK-NEXT: [[CALL2:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[ADD2]], i1 false) #[[ATTR5]]
-; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 2
-; CHECK-NEXT: [[I4:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4
-; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 2
-; CHECK-NEXT: [[I5:%.*]] = load i32, ptr [[ARRAYIDX5]], align 4
-; CHECK-NEXT: [[ADD3:%.*]] = add i32 [[I4]], [[I5]]
-; CHECK-NEXT: [[CALL3:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[ADD3]], i1 true) #[[ATTR5]]
-; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 3
-; CHECK-NEXT: [[I6:%.*]] = load i32, ptr [[ARRAYIDX6]], align 4
-; CHECK-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 3
-; CHECK-NEXT: [[I7:%.*]] = load i32, ptr [[ARRAYIDX7]], align 4
-; CHECK-NEXT: [[ADD4:%.*]] = add i32 [[I6]], [[I7]]
-; CHECK-NEXT: [[CALL4:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[ADD4]], i1 false) #[[ATTR5]]
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[A:%.*]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i32>, ptr [[B:%.*]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT: [[CALL1:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[TMP4]], i1 true) #[[ATTR5:[0-9]+]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i64 1
+; CHECK-NEXT: [[CALL2:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[TMP5]], i1 false) #[[ATTR5]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i64 2
+; CHECK-NEXT: [[CALL3:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[TMP6]], i1 true) #[[ATTR5]]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i64 3
+; CHECK-NEXT: [[CALL4:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[TMP7]], i1 false) #[[ATTR5]]
; CHECK-NEXT: store i32 [[CALL1]], ptr [[C:%.*]], align 4
; CHECK-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, ptr [[C]], i32 1
; CHECK-NEXT: store i32 [[CALL2]], ptr [[ARRAYIDX8]], align 4
@@ -304,28 +293,17 @@ entry:
define void @vec_cttz_i32_neg(ptr %a, ptr %b, ptr %c, i1) {
; CHECK-LABEL: @vec_cttz_i32_neg(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[I0:%.*]] = load i32, ptr [[A:%.*]], align 4
-; CHECK-NEXT: [[I1:%.*]] = load i32, ptr [[B:%.*]], align 4
-; CHECK-NEXT: [[ADD1:%.*]] = add i32 [[I0]], [[I1]]
-; CHECK-NEXT: [[CALL1:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[ADD1]], i1 true) #[[ATTR5]]
-; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 1
-; CHECK-NEXT: [[I2:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
-; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 1
-; CHECK-NEXT: [[I3:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4
-; CHECK-NEXT: [[ADD2:%.*]] = add i32 [[I2]], [[I3]]
-; CHECK-NEXT: [[CALL2:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[ADD2]], i1 false) #[[ATTR5]]
-; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 2
-; CHECK-NEXT: [[I4:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4
-; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 2
-; CHECK-NEXT: [[I5:%.*]] = load i32, ptr [[ARRAYIDX5]], align 4
-; CHECK-NEXT: [[ADD3:%.*]] = add i32 [[I4]], [[I5]]
-; CHECK-NEXT: [[CALL3:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[ADD3]], i1 true) #[[ATTR5]]
-; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 3
-; CHECK-NEXT: [[I6:%.*]] = load i32, ptr [[ARRAYIDX6]], align 4
-; CHECK-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 3
-; CHECK-NEXT: [[I7:%.*]] = load i32, ptr [[ARRAYIDX7]], align 4
-; CHECK-NEXT: [[ADD4:%.*]] = add i32 [[I6]], [[I7]]
-; CHECK-NEXT: [[CALL4:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[ADD4]], i1 false) #[[ATTR5]]
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[A:%.*]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i32>, ptr [[B:%.*]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT: [[CALL1:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[TMP4]], i1 true) #[[ATTR5]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i64 1
+; CHECK-NEXT: [[CALL2:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[TMP5]], i1 false) #[[ATTR5]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i64 2
+; CHECK-NEXT: [[CALL3:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[TMP6]], i1 true) #[[ATTR5]]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i64 3
+; CHECK-NEXT: [[CALL4:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[TMP7]], i1 false) #[[ATTR5]]
; CHECK-NEXT: store i32 [[CALL1]], ptr [[C:%.*]], align 4
; CHECK-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, ptr [[C]], i32 1
; CHECK-NEXT: store i32 [[CALL2]], ptr [[ARRAYIDX8]], align 4
@@ -427,28 +405,17 @@ entry:
define void @vec_powi_f32_neg(ptr %a, ptr %b, ptr %c, i32 %P, i32 %Q) {
; CHECK-LABEL: @vec_powi_f32_neg(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[I0:%.*]] = load float, ptr [[A:%.*]], align 4
-; CHECK-NEXT: [[I1:%.*]] = load float, ptr [[B:%.*]], align 4
-; CHECK-NEXT: [[ADD1:%.*]] = fadd float [[I0]], [[I1]]
-; CHECK-NEXT: [[CALL1:%.*]] = tail call float @llvm.powi.f32.i32(float [[ADD1]], i32 [[P:%.*]]) #[[ATTR5]]
-; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i32 1
-; CHECK-NEXT: [[I2:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
-; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[B]], i32 1
-; CHECK-NEXT: [[I3:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-; CHECK-NEXT: [[ADD2:%.*]] = fadd float [[I2]], [[I3]]
-; CHECK-NEXT: [[CALL2:%.*]] = tail call float @llvm.powi.f32.i32(float [[ADD2]], i32 [[Q:%.*]]) #[[ATTR5]]
-; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[A]], i32 2
-; CHECK-NEXT: [[I4:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
-; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[B]], i32 2
-; CHECK-NEXT: [[I5:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
-; CHECK-NEXT: [[ADD3:%.*]] = fadd float [[I4]], [[I5]]
-; CHECK-NEXT: [[CALL3:%.*]] = tail call float @llvm.powi.f32.i32(float [[ADD3]], i32 [[P]]) #[[ATTR5]]
-; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[A]], i32 3
-; CHECK-NEXT: [[I6:%.*]] = load float, ptr [[ARRAYIDX6]], align 4
-; CHECK-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[B]], i32 3
-; CHECK-NEXT: [[I7:%.*]] = load float, ptr [[ARRAYIDX7]], align 4
-; CHECK-NEXT: [[ADD4:%.*]] = fadd float [[I6]], [[I7]]
-; CHECK-NEXT: [[CALL4:%.*]] = tail call float @llvm.powi.f32.i32(float [[ADD4]], i32 [[Q]]) #[[ATTR5]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[A:%.*]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[B:%.*]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fadd <4 x float> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i64 0
+; CHECK-NEXT: [[CALL1:%.*]] = tail call float @llvm.powi.f32.i32(float [[TMP3]], i32 [[P:%.*]]) #[[ATTR5]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i64 1
+; CHECK-NEXT: [[CALL2:%.*]] = tail call float @llvm.powi.f32.i32(float [[TMP4]], i32 [[Q:%.*]]) #[[ATTR5]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i64 2
+; CHECK-NEXT: [[CALL3:%.*]] = tail call float @llvm.powi.f32.i32(float [[TMP5]], i32 [[P]]) #[[ATTR5]]
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i64 3
+; CHECK-NEXT: [[CALL4:%.*]] = tail call float @llvm.powi.f32.i32(float [[TMP6]], i32 [[Q]]) #[[ATTR5]]
; CHECK-NEXT: store float [[CALL1]], ptr [[C:%.*]], align 4
; CHECK-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[C]], i32 1
; CHECK-NEXT: store float [[CALL2]], ptr [[ARRAYIDX8]], align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/lookahead.ll b/llvm/test/Transforms/SLPVectorizer/X86/lookahead.ll
index c926ef716b2a3..ba15324f30e51 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/lookahead.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/lookahead.ll
@@ -590,12 +590,11 @@ define double @splat_loads(ptr %array1, ptr %array2, ptr %ptrA, ptr %ptrB) {
; SSE-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[ARRAY2:%.*]], align 8
; SSE-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
; SSE-NEXT: [[TMP4:%.*]] = fmul <4 x double> [[TMP2]], [[TMP3]]
-; SSE-NEXT: [[TMP6:%.*]] = extractelement <4 x double> [[TMP4]], i64 0
-; SSE-NEXT: [[TMP7:%.*]] = extractelement <4 x double> [[TMP4]], i64 2
-; SSE-NEXT: [[ADD3:%.*]] = fadd double [[TMP6]], [[TMP7]]
-; SSE-NEXT: [[TMP9:%.*]] = extractelement <4 x double> [[TMP4]], i64 1
-; SSE-NEXT: [[TMP8:%.*]] = extractelement <4 x double> [[TMP4]], i64 3
-; SSE-NEXT: [[ADD2:%.*]] = fadd double [[TMP9]], [[TMP8]]
+; SSE-NEXT: [[TMP5:%.*]] = shufflevector <4 x double> [[TMP4]], <4 x double> poison, <2 x i32> <i32 0, i32 1>
+; SSE-NEXT: [[TMP6:%.*]] = shufflevector <4 x double> [[TMP4]], <4 x double> poison, <2 x i32> <i32 2, i32 3>
+; SSE-NEXT: [[TMP7:%.*]] = fadd <2 x double> [[TMP5]], [[TMP6]]
+; SSE-NEXT: [[ADD3:%.*]] = extractelement <2 x double> [[TMP7]], i64 0
+; SSE-NEXT: [[ADD2:%.*]] = extractelement <2 x double> [[TMP7]], i64 1
; SSE-NEXT: [[ADD4:%.*]] = fadd double [[ADD3]], [[ADD2]]
; SSE-NEXT: ret double [[ADD4]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/non-vectorizable-inst-operand.ll b/llvm/test/Transforms/SLPVectorizer/X86/non-vectorizable-inst-operand.ll
index 262ab83f2698d..f1a85aede935c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/non-vectorizable-inst-operand.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/non-vectorizable-inst-operand.ll
@@ -108,14 +108,13 @@ define void @test_two_invokes_with_vectorizable_operands(ptr %p, ptr %out0, ptr
; CHECK-LABEL: define void @test_two_invokes_with_vectorizable_operands(
; CHECK-SAME: ptr [[P:%.*]], ptr [[OUT0:%.*]], ptr [[OUT1:%.*]]) #[[ATTR0]] personality ptr @__gxx_personality_v0 {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[P]], align 8
-; CHECK-NEXT: [[P1:%.*]] = getelementptr double, ptr [[P]], i64 1
-; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[P1]], align 8
-; CHECK-NEXT: [[M0:%.*]] = fmul double [[A0]], 2.000000e+00
-; CHECK-NEXT: [[M1:%.*]] = fmul double [[A1]], 3.000000e+00
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = fmul <2 x double> [[TMP0]], <double 2.000000e+00, double 3.000000e+00>
+; CHECK-NEXT: [[M0:%.*]] = extractelement <2 x double> [[TMP1]], i64 0
; CHECK-NEXT: [[R0:%.*]] = invoke double @user_func(double [[M0]])
; CHECK-NEXT: to label %[[CONT1:.*]] unwind label %[[EH:.*]]
; CHECK: [[CONT1]]:
+; CHECK-NEXT: [[M1:%.*]] = extractelement <2 x double> [[TMP1]], i64 1
; CHECK-NEXT: [[R1:%.*]] = invoke double @user_func(double [[M1]])
; CHECK-NEXT: to label %[[CONT2:.*]] unwind label %[[EH]]
; CHECK: [[CONT2]]:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/parent-node-split-non-schedulable.ll b/llvm/test/Transforms/SLPVectorizer/X86/parent-node-split-non-schedulable.ll
index fcd009961a754..537443a78926b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/parent-node-split-non-schedulable.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/parent-node-split-non-schedulable.ll
@@ -24,16 +24,18 @@ define i32 @main(ptr %c, i32 %0, i1 %tobool4.not, i16 %1) {
; CHECK-NEXT: br label %[[IF_END14]]
; CHECK: [[IF_END14]]:
; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[C]], align 4
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[TMP3]], i64 1
-; CHECK-NEXT: [[AND:%.*]] = and i32 [[TMP8]], 1
-; CHECK-NEXT: [[AND25:%.*]] = and i32 [[TMP0]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <2 x i32> <i32 1, i32 poison>
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP0]], i64 1
+; CHECK-NEXT: [[TMP24:%.*]] = and <2 x i32> [[TMP13]], splat (i32 1)
+; CHECK-NEXT: [[AND:%.*]] = extractelement <2 x i32> [[TMP24]], i64 0
; CHECK-NEXT: [[NOT:%.*]] = xor i32 [[AND]], 1
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[TMP3]], i64 0
; CHECK-NEXT: [[AND17:%.*]] = and i32 [[TMP9]], 1
; CHECK-NEXT: [[DIV20:%.*]] = sdiv i32 [[AND17]], [[TMP0]]
; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[AND17]], i64 1
; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP10]], i64 2
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[AND25]], i64 3
+; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <2 x i32> [[TMP24]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> [[TMP27]], <4 x i32> <i32 0, i32 1, i32 2, i32 5>
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> <i32 0, i32 1, i32 poison, i32 1>, i32 [[DIV20]], i64 2
; CHECK-NEXT: [[TMP16:%.*]] = xor <4 x i32> [[TMP14]], [[TMP15]]
; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi-comparator-fix-vec-ops-compare.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi-comparator-fix-vec-ops-compare.ll
index ed36843f60fdc..56720be9633e0 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/phi-comparator-fix-vec-ops-compare.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/phi-comparator-fix-vec-ops-compare.ll
@@ -6,19 +6,17 @@ define void @test({ <2 x float>, float } %0, <2 x float> %1, i1 %2) {
; CHECK-SAME: { <2 x float>, float } [[TMP0:%.*]], <2 x float> [[TMP1:%.*]], i1 [[TMP2:%.*]]) {
; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <2 x float>, float } [[TMP0]], 0
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x float> [[TMP4]], i64 0
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x float> [[TMP1]], i64 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x float> [[TMP1]], i64 0
; CHECK-NEXT: br i1 [[TMP2]], label %[[BB9:.*]], label %[[BB8:.*]]
; CHECK: [[BB8]]:
; CHECK-NEXT: br label %[[BB9]]
; CHECK: [[BB9]]:
-; CHECK-NEXT: [[TMP10:%.*]] = phi float [ 0.000000e+00, %[[BB8]] ], [ [[TMP7]], [[TMP3:%.*]] ]
-; CHECK-NEXT: [[TMP15:%.*]] = phi float [ 0.000000e+00, %[[BB8]] ], [ [[TMP6]], [[TMP3]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = phi float [ 0.000000e+00, %[[BB8]] ], [ [[TMP5]], [[TMP3]] ]
-; CHECK-NEXT: [[TMP14:%.*]] = fpext float [[TMP11]] to double
+; CHECK-NEXT: [[TMP15:%.*]] = phi float [ 0.000000e+00, %[[BB8]] ], [ [[TMP5]], [[TMP3:%.*]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x float> [ zeroinitializer, %[[BB8]] ], [ [[TMP1]], [[TMP3]] ]
; CHECK-NEXT: [[TMP12:%.*]] = fpext float [[TMP15]] to double
-; CHECK-NEXT: [[TMP13:%.*]] = tail call i32 (ptr, ptr, ...) @fprintf(ptr null, ptr null, double [[TMP14]], double [[TMP12]], double 0.000000e+00)
-; CHECK-NEXT: [[TMP16:%.*]] = fpext float [[TMP10]] to double
+; CHECK-NEXT: [[TMP11:%.*]] = fpext <2 x float> [[TMP9]] to <2 x double>
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[TMP11]], i64 1
+; CHECK-NEXT: [[TMP13:%.*]] = tail call i32 (ptr, ptr, ...) @fprintf(ptr null, ptr null, double [[TMP12]], double [[TMP14]], double 0.000000e+00)
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x double> [[TMP11]], i64 0
; CHECK-NEXT: [[TMP17:%.*]] = tail call i32 (ptr, ptr, ...) @fprintf(ptr null, ptr null, double 0.000000e+00, double [[TMP16]], double 0.000000e+00)
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr42022-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr42022-inseltpoison.ll
index b5cfb846e3d49..3e6b4ca54b4ce 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr42022-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr42022-inseltpoison.ll
@@ -122,21 +122,16 @@ define {%StructTy, %StructTy} @StructOfStruct(ptr %Ptr) {
define {%StructTy, float, float} @NonHomogeneousStruct(ptr %Ptr) {
; CHECK-LABEL: @NonHomogeneousStruct(
-; CHECK-NEXT: [[L0:%.*]] = load float, ptr [[PTR:%.*]], align 4
-; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds float, ptr [[PTR]], i64 1
-; CHECK-NEXT: [[L1:%.*]] = load float, ptr [[GEP1]], align 4
-; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds float, ptr [[PTR]], i64 2
-; CHECK-NEXT: [[L2:%.*]] = load float, ptr [[GEP2]], align 4
-; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds float, ptr [[PTR]], i64 3
-; CHECK-NEXT: [[L3:%.*]] = load float, ptr [[GEP3]], align 4
-; CHECK-NEXT: [[FADD0:%.*]] = fadd fast float [[L0]], 1.100000e+01
-; CHECK-NEXT: [[FADD1:%.*]] = fadd fast float [[L1]], 1.200000e+01
-; CHECK-NEXT: [[FADD2:%.*]] = fadd fast float [[L2]], 1.300000e+01
-; CHECK-NEXT: [[FADD3:%.*]] = fadd fast float [[L3]], 1.400000e+01
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[PTR:%.*]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fadd fast <4 x float> [[TMP1]], <float 1.100000e+01, float 1.200000e+01, float 1.300000e+01, float 1.400000e+01>
+; CHECK-NEXT: [[FADD0:%.*]] = extractelement <4 x float> [[TMP2]], i64 0
; CHECK-NEXT: [[STRUCTIN0:%.*]] = insertvalue [[STRUCTTY:%.*]] undef, float [[FADD0]], 0
+; CHECK-NEXT: [[FADD1:%.*]] = extractelement <4 x float> [[TMP2]], i64 1
; CHECK-NEXT: [[STRUCTIN1:%.*]] = insertvalue [[STRUCTTY]] [[STRUCTIN0]], float [[FADD1]], 1
; CHECK-NEXT: [[RET0:%.*]] = insertvalue { [[STRUCTTY]], float, float } undef, [[STRUCTTY]] [[STRUCTIN1]], 0
+; CHECK-NEXT: [[FADD2:%.*]] = extractelement <4 x float> [[TMP2]], i64 2
; CHECK-NEXT: [[RET1:%.*]] = insertvalue { [[STRUCTTY]], float, float } [[RET0]], float [[FADD2]], 1
+; CHECK-NEXT: [[FADD3:%.*]] = extractelement <4 x float> [[TMP2]], i64 3
; CHECK-NEXT: [[RET2:%.*]] = insertvalue { [[STRUCTTY]], float, float } [[RET1]], float [[FADD3]], 2
; CHECK-NEXT: ret { [[STRUCTTY]], float, float } [[RET2]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll
index 6bd1b64b88a57..6f56a507e7ab0 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr42022.ll
@@ -122,21 +122,16 @@ define {%StructTy, %StructTy} @StructOfStruct(ptr %Ptr) {
define {%StructTy, float, float} @NonHomogeneousStruct(ptr %Ptr) {
; CHECK-LABEL: @NonHomogeneousStruct(
-; CHECK-NEXT: [[L0:%.*]] = load float, ptr [[PTR:%.*]], align 4
-; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds float, ptr [[PTR]], i64 1
-; CHECK-NEXT: [[L1:%.*]] = load float, ptr [[GEP1]], align 4
-; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds float, ptr [[PTR]], i64 2
-; CHECK-NEXT: [[L2:%.*]] = load float, ptr [[GEP2]], align 4
-; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds float, ptr [[PTR]], i64 3
-; CHECK-NEXT: [[L3:%.*]] = load float, ptr [[GEP3]], align 4
-; CHECK-NEXT: [[FADD0:%.*]] = fadd fast float [[L0]], 1.100000e+01
-; CHECK-NEXT: [[FADD1:%.*]] = fadd fast float [[L1]], 1.200000e+01
-; CHECK-NEXT: [[FADD2:%.*]] = fadd fast float [[L2]], 1.300000e+01
-; CHECK-NEXT: [[FADD3:%.*]] = fadd fast float [[L3]], 1.400000e+01
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[PTR:%.*]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fadd fast <4 x float> [[TMP1]], <float 1.100000e+01, float 1.200000e+01, float 1.300000e+01, float 1.400000e+01>
+; CHECK-NEXT: [[FADD0:%.*]] = extractelement <4 x float> [[TMP2]], i64 0
; CHECK-NEXT: [[STRUCTIN0:%.*]] = insertvalue [[STRUCTTY:%.*]] undef, float [[FADD0]], 0
+; CHECK-NEXT: [[FADD1:%.*]] = extractelement <4 x float> [[TMP2]], i64 1
; CHECK-NEXT: [[STRUCTIN1:%.*]] = insertvalue [[STRUCTTY]] [[STRUCTIN0]], float [[FADD1]], 1
; CHECK-NEXT: [[RET0:%.*]] = insertvalue { [[STRUCTTY]], float, float } undef, [[STRUCTTY]] [[STRUCTIN1]], 0
+; CHECK-NEXT: [[FADD2:%.*]] = extractelement <4 x float> [[TMP2]], i64 2
; CHECK-NEXT: [[RET1:%.*]] = insertvalue { [[STRUCTTY]], float, float } [[RET0]], float [[FADD2]], 1
+; CHECK-NEXT: [[FADD3:%.*]] = extractelement <4 x float> [[TMP2]], i64 3
; CHECK-NEXT: [[RET2:%.*]] = insertvalue { [[STRUCTTY]], float, float } [[RET1]], float [[FADD3]], 2
; CHECK-NEXT: ret { [[STRUCTTY]], float, float } [[RET2]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr52275.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr52275.ll
index 3f2211ce31806..27e394a32b676 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr52275.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr52275.ll
@@ -43,13 +43,10 @@ define <2 x i8> @test2(<2 x i8> %t6, ptr %t1) {
; CHECK-NEXT: ret <2 x i8> [[T11]]
;
; FORCE_SLP-LABEL: @test2(
-; FORCE_SLP-NEXT: [[T3:%.*]] = load i32, ptr [[T1:%.*]], align 4
-; FORCE_SLP-NEXT: [[T4:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 1
-; FORCE_SLP-NEXT: [[T5:%.*]] = load i32, ptr [[T4]], align 4
-; FORCE_SLP-NEXT: [[T7:%.*]] = trunc i32 [[T3]] to i8
-; FORCE_SLP-NEXT: [[T8:%.*]] = insertelement <2 x i8> [[T6:%.*]], i8 [[T7]], i64 0
-; FORCE_SLP-NEXT: [[T9:%.*]] = trunc i32 [[T5]] to i8
-; FORCE_SLP-NEXT: [[T10:%.*]] = insertelement <2 x i8> [[T8]], i8 [[T9]], i64 1
+; FORCE_SLP-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[T1:%.*]], align 4
+; FORCE_SLP-NEXT: [[TMP2:%.*]] = trunc <2 x i32> [[TMP1]] to <2 x i8>
+; FORCE_SLP-NEXT: [[T8:%.*]] = shufflevector <2 x i8> [[T6:%.*]], <2 x i8> [[TMP2]], <2 x i32> <i32 2, i32 1>
+; FORCE_SLP-NEXT: [[T10:%.*]] = shufflevector <2 x i8> [[T8]], <2 x i8> [[TMP2]], <2 x i32> <i32 0, i32 3>
; FORCE_SLP-NEXT: [[T11:%.*]] = add <2 x i8> [[T10]], [[T8]]
; FORCE_SLP-NEXT: ret <2 x i8> [[T11]]
;
@@ -77,13 +74,10 @@ define <2 x i8> @test_reorder(<2 x i8> %t6, ptr %t1) {
; CHECK-NEXT: ret <2 x i8> [[T11]]
;
; FORCE_SLP-LABEL: @test_reorder(
-; FORCE_SLP-NEXT: [[T3:%.*]] = load i32, ptr [[T1:%.*]], align 4
-; FORCE_SLP-NEXT: [[T4:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 1
-; FORCE_SLP-NEXT: [[T5:%.*]] = load i32, ptr [[T4]], align 4
-; FORCE_SLP-NEXT: [[T7:%.*]] = trunc i32 [[T3]] to i8
-; FORCE_SLP-NEXT: [[T8:%.*]] = insertelement <2 x i8> [[T6:%.*]], i8 [[T7]], i64 1
-; FORCE_SLP-NEXT: [[T9:%.*]] = trunc i32 [[T5]] to i8
-; FORCE_SLP-NEXT: [[T10:%.*]] = insertelement <2 x i8> [[T8]], i8 [[T9]], i64 0
+; FORCE_SLP-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[T1:%.*]], align 4
+; FORCE_SLP-NEXT: [[TMP2:%.*]] = trunc <2 x i32> [[TMP1]] to <2 x i8>
+; FORCE_SLP-NEXT: [[T8:%.*]] = shufflevector <2 x i8> [[T6:%.*]], <2 x i8> [[TMP2]], <2 x i32> <i32 0, i32 2>
+; FORCE_SLP-NEXT: [[T10:%.*]] = shufflevector <2 x i8> [[T8]], <2 x i8> [[TMP2]], <2 x i32> <i32 3, i32 1>
; FORCE_SLP-NEXT: [[T11:%.*]] = add <2 x i8> [[T10]], [[T8]]
; FORCE_SLP-NEXT: ret <2 x i8> [[T11]]
;
@@ -111,13 +105,11 @@ define <4 x i8> @test_subvector(<4 x i8> %t6, ptr %t1) {
; CHECK-NEXT: ret <4 x i8> [[T11]]
;
; FORCE_SLP-LABEL: @test_subvector(
-; FORCE_SLP-NEXT: [[T3:%.*]] = load i32, ptr [[T1:%.*]], align 4
-; FORCE_SLP-NEXT: [[T4:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 1
-; FORCE_SLP-NEXT: [[T5:%.*]] = load i32, ptr [[T4]], align 4
-; FORCE_SLP-NEXT: [[T7:%.*]] = trunc i32 [[T3]] to i8
-; FORCE_SLP-NEXT: [[T8:%.*]] = insertelement <4 x i8> [[T6:%.*]], i8 [[T7]], i64 0
-; FORCE_SLP-NEXT: [[T9:%.*]] = trunc i32 [[T5]] to i8
-; FORCE_SLP-NEXT: [[T10:%.*]] = insertelement <4 x i8> [[T8]], i8 [[T9]], i64 1
+; FORCE_SLP-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[T1:%.*]], align 4
+; FORCE_SLP-NEXT: [[TMP2:%.*]] = trunc <2 x i32> [[TMP1]] to <2 x i8>
+; FORCE_SLP-NEXT: [[TMP3:%.*]] = shufflevector <2 x i8> [[TMP2]], <2 x i8> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; FORCE_SLP-NEXT: [[T8:%.*]] = shufflevector <4 x i8> [[T6:%.*]], <4 x i8> [[TMP3]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+; FORCE_SLP-NEXT: [[T10:%.*]] = shufflevector <4 x i8> [[T8]], <4 x i8> [[TMP3]], <4 x i32> <i32 0, i32 5, i32 2, i32 3>
; FORCE_SLP-NEXT: [[T11:%.*]] = add <4 x i8> [[T10]], [[T8]]
; FORCE_SLP-NEXT: ret <4 x i8> [[T11]]
;
@@ -145,13 +137,11 @@ define <4 x i8> @test_subvector_reorder(<4 x i8> %t6, ptr %t1) {
; CHECK-NEXT: ret <4 x i8> [[T11]]
;
; FORCE_SLP-LABEL: @test_subvector_reorder(
-; FORCE_SLP-NEXT: [[T3:%.*]] = load i32, ptr [[T1:%.*]], align 4
-; FORCE_SLP-NEXT: [[T4:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 1
-; FORCE_SLP-NEXT: [[T5:%.*]] = load i32, ptr [[T4]], align 4
-; FORCE_SLP-NEXT: [[T7:%.*]] = trunc i32 [[T3]] to i8
-; FORCE_SLP-NEXT: [[T8:%.*]] = insertelement <4 x i8> [[T6:%.*]], i8 [[T7]], i64 3
-; FORCE_SLP-NEXT: [[T9:%.*]] = trunc i32 [[T5]] to i8
-; FORCE_SLP-NEXT: [[T10:%.*]] = insertelement <4 x i8> [[T8]], i8 [[T9]], i64 2
+; FORCE_SLP-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[T1:%.*]], align 4
+; FORCE_SLP-NEXT: [[TMP2:%.*]] = trunc <2 x i32> [[TMP1]] to <2 x i8>
+; FORCE_SLP-NEXT: [[TMP3:%.*]] = shufflevector <2 x i8> [[TMP2]], <2 x i8> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; FORCE_SLP-NEXT: [[T8:%.*]] = shufflevector <4 x i8> [[T6:%.*]], <4 x i8> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+; FORCE_SLP-NEXT: [[T10:%.*]] = shufflevector <4 x i8> [[T8]], <4 x i8> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 5, i32 3>
; FORCE_SLP-NEXT: [[T11:%.*]] = add <4 x i8> [[T10]], [[T8]]
; FORCE_SLP-NEXT: ret <4 x i8> [[T11]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reorder_diamond_match.ll b/llvm/test/Transforms/SLPVectorizer/X86/reorder_diamond_match.ll
index e12c8623932ef..3fa118b5a1c76 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reorder_diamond_match.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reorder_diamond_match.ll
@@ -4,38 +4,26 @@
define void @test() {
; CHECK-LABEL: @test(
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr undef, i64 4
-; CHECK-NEXT: [[TMP2:%.*]] = load i8, ptr [[TMP1]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = zext i8 [[TMP2]] to i32
-; CHECK-NEXT: [[TMP4:%.*]] = sub nsw i32 0, [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = shl nsw i32 [[TMP4]], 0
-; CHECK-NEXT: [[TMP6:%.*]] = add nsw i32 [[TMP5]], 0
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr undef, i64 5
-; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[TMP7]], align 1
-; CHECK-NEXT: [[TMP9:%.*]] = zext i8 [[TMP8]] to i32
-; CHECK-NEXT: [[TMP10:%.*]] = sub nsw i32 0, [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = shl nsw i32 [[TMP10]], 0
-; CHECK-NEXT: [[TMP12:%.*]] = add nsw i32 [[TMP11]], 0
-; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr undef, i64 6
-; CHECK-NEXT: [[TMP14:%.*]] = load i8, ptr [[TMP13]], align 1
-; CHECK-NEXT: [[TMP15:%.*]] = zext i8 [[TMP14]] to i32
-; CHECK-NEXT: [[TMP16:%.*]] = sub nsw i32 0, [[TMP15]]
-; CHECK-NEXT: [[TMP17:%.*]] = shl nsw i32 [[TMP16]], 0
-; CHECK-NEXT: [[TMP18:%.*]] = add nsw i32 [[TMP17]], 0
-; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i8, ptr undef, i64 7
-; CHECK-NEXT: [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
-; CHECK-NEXT: [[TMP21:%.*]] = zext i8 [[TMP20]] to i32
-; CHECK-NEXT: [[TMP22:%.*]] = sub nsw i32 0, [[TMP21]]
-; CHECK-NEXT: [[TMP23:%.*]] = shl nsw i32 [[TMP22]], 0
-; CHECK-NEXT: [[TMP24:%.*]] = add nsw i32 [[TMP23]], 0
-; CHECK-NEXT: [[TMP25:%.*]] = add nsw i32 [[TMP12]], [[TMP6]]
-; CHECK-NEXT: [[TMP26:%.*]] = sub nsw i32 [[TMP6]], [[TMP12]]
-; CHECK-NEXT: [[TMP27:%.*]] = add nsw i32 [[TMP24]], [[TMP18]]
-; CHECK-NEXT: [[TMP28:%.*]] = sub nsw i32 [[TMP18]], [[TMP24]]
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: [[TMP4:%.*]] = zext <2 x i8> [[TMP3]] to <2 x i32>
+; CHECK-NEXT: [[TMP5:%.*]] = sub nsw <2 x i32> zeroinitializer, [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = shl nsw <2 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = add nsw <2 x i32> [[TMP6]], zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT: [[TMP9:%.*]] = zext <2 x i8> [[TMP8]] to <2 x i32>
+; CHECK-NEXT: [[TMP10:%.*]] = sub nsw <2 x i32> zeroinitializer, [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = shl nsw <2 x i32> [[TMP10]], zeroinitializer
+; CHECK-NEXT: [[TMP12:%.*]] = add nsw <2 x i32> [[TMP11]], zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = add nsw <2 x i32> [[TMP12]], [[TMP7]]
+; CHECK-NEXT: [[TMP14:%.*]] = sub nsw <2 x i32> [[TMP7]], [[TMP12]]
; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds [4 x [4 x i32]], ptr undef, i64 0, i64 1, i64 0
-; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x i32> <i32 poison, i32 poison, i32 0, i32 0>, i32 [[TMP25]], i64 0
-; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> [[TMP30]], i32 [[TMP26]], i64 1
-; CHECK-NEXT: [[TMP32:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 poison, i32 poison>, i32 [[TMP27]], i64 2
-; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x i32> [[TMP32]], i32 [[TMP28]], i64 3
+; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <2 x i32> [[TMP13]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <4 x i32> <i32 poison, i32 poison, i32 0, i32 0>, <4 x i32> [[TMP16]], <4 x i32> <i32 4, i32 poison, i32 2, i32 3>
+; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <2 x i32> [[TMP14]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP31:%.*]] = shufflevector <4 x i32> [[TMP17]], <4 x i32> [[TMP18]], <4 x i32> <i32 0, i32 4, i32 2, i32 3>
+; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <4 x i32> <i32 0, i32 0, i32 poison, i32 poison>, <4 x i32> [[TMP16]], <4 x i32> <i32 0, i32 1, i32 5, i32 poison>
+; CHECK-NEXT: [[TMP33:%.*]] = shufflevector <4 x i32> [[TMP20]], <4 x i32> [[TMP18]], <4 x i32> <i32 0, i32 1, i32 2, i32 5>
; CHECK-NEXT: [[TMP34:%.*]] = sub nsw <4 x i32> [[TMP31]], [[TMP33]]
; CHECK-NEXT: store <4 x i32> [[TMP34]], ptr [[TMP29]], align 16
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reused-mask-with-poison-index.ll b/llvm/test/Transforms/SLPVectorizer/X86/reused-mask-with-poison-index.ll
index b36e6e649b687..42229a6dc3bcd 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reused-mask-with-poison-index.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reused-mask-with-poison-index.ll
@@ -8,24 +8,25 @@ define fastcc i32 @test(ptr %0, <2 x float> %1, i1 %2, float %3, float %4) {
; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x float> [[TMP1]], i64 1
; CHECK-NEXT: br label %[[BB8:.*]]
; CHECK: [[BB8]]:
-; CHECK-NEXT: [[TMP9:%.*]] = phi float [ 0.000000e+00, [[TMP5:%.*]] ], [ [[TMP58:%.*]], %[[TMP56:.*]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = phi float [ 0.000000e+00, [[TMP5]] ], [ [[TMP59:%.*]], %[[TMP56]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = phi float [ [[TMP4]], [[TMP5]] ], [ [[TMP60:%.*]], %[[TMP56]] ]
-; CHECK-NEXT: [[TMP12:%.*]] = phi float [ [[TMP7]], [[TMP5]] ], [ [[TMP61:%.*]], %[[TMP56]] ]
-; CHECK-NEXT: [[TMP13:%.*]] = phi float [ [[TMP6]], [[TMP5]] ], [ [[TMP62:%.*]], %[[TMP56]] ]
-; CHECK-NEXT: [[TMP14:%.*]] = phi float [ 0.000000e+00, [[TMP5]] ], [ [[TMP63:%.*]], %[[TMP56]] ]
-; CHECK-NEXT: [[TMP17:%.*]] = phi float [ 0.000000e+00, [[TMP5]] ], [ [[TMP64:%.*]], %[[TMP56]] ]
-; CHECK-NEXT: [[TMP16:%.*]] = phi float [ 0.000000e+00, [[TMP5]] ], [ [[TMP65:%.*]], %[[TMP56]] ]
-; CHECK-NEXT: [[TMP15:%.*]] = phi float [ undef, [[TMP5]] ], [ [[TMP66:%.*]], %[[TMP56]] ]
-; CHECK-NEXT: [[TMP18:%.*]] = phi float [ 0.000000e+00, [[TMP5]] ], [ [[TMP67:%.*]], %[[TMP56]] ]
-; CHECK-NEXT: [[TMP19:%.*]] = phi float [ [[TMP4]], [[TMP5]] ], [ [[TMP68:%.*]], %[[TMP56]] ]
-; CHECK-NEXT: [[TMP20:%.*]] = phi float [ [[TMP4]], [[TMP5]] ], [ [[TMP69:%.*]], %[[TMP56]] ]
-; CHECK-NEXT: [[TMP21:%.*]] = phi float [ [[TMP4]], [[TMP5]] ], [ [[TMP70:%.*]], %[[TMP56]] ]
-; CHECK-NEXT: [[TMP22:%.*]] = phi float [ [[TMP4]], [[TMP5]] ], [ [[TMP71:%.*]], %[[TMP56]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi float [ 0.000000e+00, [[TMP5:%.*]] ], [ [[TMP57:%.*]], %[[TMP56:.*]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = phi float [ 0.000000e+00, [[TMP5]] ], [ [[TMP58:%.*]], %[[TMP56]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi float [ [[TMP4]], [[TMP5]] ], [ [[TMP59:%.*]], %[[TMP56]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = phi float [ [[TMP7]], [[TMP5]] ], [ [[TMP60:%.*]], %[[TMP56]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = phi float [ [[TMP6]], [[TMP5]] ], [ [[TMP61:%.*]], %[[TMP56]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = phi float [ 0.000000e+00, [[TMP5]] ], [ [[TMP62:%.*]], %[[TMP56]] ]
+; CHECK-NEXT: [[TMP15:%.*]] = phi float [ 0.000000e+00, [[TMP5]] ], [ [[TMP63:%.*]], %[[TMP56]] ]
+; CHECK-NEXT: [[TMP16:%.*]] = phi float [ 0.000000e+00, [[TMP5]] ], [ [[TMP64:%.*]], %[[TMP56]] ]
+; CHECK-NEXT: [[TMP17:%.*]] = phi float [ undef, [[TMP5]] ], [ [[TMP65:%.*]], %[[TMP56]] ]
+; CHECK-NEXT: [[TMP18:%.*]] = phi float [ 0.000000e+00, [[TMP5]] ], [ [[TMP66:%.*]], %[[TMP56]] ]
+; CHECK-NEXT: [[TMP19:%.*]] = phi float [ [[TMP4]], [[TMP5]] ], [ [[TMP67:%.*]], %[[TMP56]] ]
+; CHECK-NEXT: [[TMP20:%.*]] = phi float [ [[TMP4]], [[TMP5]] ], [ [[TMP68:%.*]], %[[TMP56]] ]
+; CHECK-NEXT: [[TMP21:%.*]] = phi float [ [[TMP4]], [[TMP5]] ], [ [[TMP69:%.*]], %[[TMP56]] ]
+; CHECK-NEXT: [[TMP22:%.*]] = phi float [ [[TMP4]], [[TMP5]] ], [ [[TMP70:%.*]], %[[TMP56]] ]
; CHECK-NEXT: [[TMP23:%.*]] = phi float [ 0.000000e+00, [[TMP5]] ], [ [[TMP73:%.*]], %[[TMP56]] ]
; CHECK-NEXT: [[TMP24:%.*]] = phi float [ 0.000000e+00, [[TMP5]] ], [ [[TMP72:%.*]], %[[TMP56]] ]
; CHECK-NEXT: [[TMP25:%.*]] = phi <4 x float> [ zeroinitializer, [[TMP5]] ], [ poison, %[[TMP56]] ]
; CHECK-NEXT: [[TMP26:%.*]] = phi <2 x float> [ zeroinitializer, [[TMP5]] ], [ poison, %[[TMP56]] ]
+; CHECK-NEXT: [[TMP27:%.*]] = phi <2 x float> [ <float undef, float 0.000000e+00>, [[TMP5]] ], [ poison, %[[TMP56]] ]
; CHECK-NEXT: br i1 false, label %[[BB57:.*]], label %[[BB27:.*]]
; CHECK: [[BB27]]:
; CHECK-NEXT: [[TMP28:%.*]] = fcmp olt float [[TMP22]], 0.000000e+00
@@ -37,15 +38,13 @@ define fastcc i32 @test(ptr %0, <2 x float> %1, i1 %2, float %3, float %4) {
; CHECK-NEXT: [[TMP32:%.*]] = insertelement <4 x float> poison, float [[TMP3]], i64 0
; CHECK-NEXT: [[TMP33:%.*]] = shufflevector <4 x float> [[TMP32]], <4 x float> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP34:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP33]], <4 x float> zeroinitializer, <4 x float> zeroinitializer)
-; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x float> [[TMP25]], i64 0
-; CHECK-NEXT: [[TMP38:%.*]] = fsub float [[TMP15]], [[TMP37]]
-; CHECK-NEXT: [[TMP48:%.*]] = extractelement <4 x float> [[TMP25]], i64 1
-; CHECK-NEXT: [[TMP49:%.*]] = fsub float [[TMP17]], [[TMP48]]
+; CHECK-NEXT: [[TMP36:%.*]] = shufflevector <4 x float> [[TMP25]], <4 x float> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP37:%.*]] = fsub <2 x float> [[TMP27]], [[TMP36]]
; CHECK-NEXT: [[TMP39:%.*]] = insertelement <2 x float> zeroinitializer, float [[TMP14]], i64 0
; CHECK-NEXT: [[TMP40:%.*]] = shufflevector <4 x float> [[TMP25]], <4 x float> poison, <2 x i32> <i32 poison, i32 2>
; CHECK-NEXT: [[TMP41:%.*]] = shufflevector <2 x float> [[TMP39]], <2 x float> [[TMP40]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP42:%.*]] = insertelement <2 x float> zeroinitializer, float [[TMP49]], i64 0
-; CHECK-NEXT: [[TMP43:%.*]] = insertelement <2 x float> zeroinitializer, float [[TMP38]], i64 0
+; CHECK-NEXT: [[TMP42:%.*]] = shufflevector <2 x float> zeroinitializer, <2 x float> [[TMP37]], <2 x i32> <i32 3, i32 1>
+; CHECK-NEXT: [[TMP43:%.*]] = shufflevector <2 x float> zeroinitializer, <2 x float> [[TMP37]], <2 x i32> <i32 2, i32 1>
; CHECK-NEXT: [[TMP44:%.*]] = fmul <2 x float> [[TMP42]], [[TMP43]]
; CHECK-NEXT: [[TMP45:%.*]] = tail call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP41]], <2 x float> [[TMP26]], <2 x float> [[TMP44]])
; CHECK-NEXT: [[TMP46:%.*]] = extractelement <2 x float> [[TMP45]], i64 0
@@ -56,8 +55,8 @@ define fastcc i32 @test(ptr %0, <2 x float> %1, i1 %2, float %3, float %4) {
; CHECK: [[BB49:.*]]:
; CHECK-NEXT: br label %[[BB50]]
; CHECK: [[BB50]]:
-; CHECK-NEXT: [[TMP51:%.*]] = phi float [ [[TMP12]], %[[BB49]] ], [ [[TMP19]], %[[BB48]] ], [ 0.000000e+00, %[[BB31]] ]
-; CHECK-NEXT: [[TMP52:%.*]] = phi float [ [[TMP13]], %[[BB49]] ], [ [[TMP20]], %[[BB48]] ], [ 0.000000e+00, %[[BB31]] ]
+; CHECK-NEXT: [[TMP50:%.*]] = phi float [ [[TMP12]], %[[BB49]] ], [ [[TMP19]], %[[BB48]] ], [ 0.000000e+00, %[[BB31]] ]
+; CHECK-NEXT: [[TMP51:%.*]] = phi float [ [[TMP13]], %[[BB49]] ], [ [[TMP20]], %[[BB48]] ], [ 0.000000e+00, %[[BB31]] ]
; CHECK-NEXT: br i1 [[TMP2]], label %[[BB57]], label %[[BB53:.*]]
; CHECK: [[BB53]]:
; CHECK-NEXT: [[TMP54:%.*]] = extractelement <2 x float> [[TMP1]], i64 0
@@ -66,20 +65,20 @@ define fastcc i32 @test(ptr %0, <2 x float> %1, i1 %2, float %3, float %4) {
; CHECK: [[TMP56]]:
; CHECK-NEXT: br label %[[BB8]]
; CHECK: [[BB57]]:
-; CHECK-NEXT: [[TMP58]] = phi float [ [[TMP9]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP3]], %[[BB50]] ]
-; CHECK-NEXT: [[TMP59]] = phi float [ [[TMP10]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ [[TMP55]], %[[BB53]] ], [ 0.000000e+00, %[[BB50]] ]
-; CHECK-NEXT: [[TMP60]] = phi float [ [[TMP11]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP3]], %[[BB50]] ]
-; CHECK-NEXT: [[TMP61]] = phi float [ [[TMP12]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ 0.000000e+00, %[[BB50]] ]
-; CHECK-NEXT: [[TMP62]] = phi float [ [[TMP13]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ [[TMP54]], %[[BB53]] ], [ 0.000000e+00, %[[BB50]] ]
-; CHECK-NEXT: [[TMP63]] = phi float [ [[TMP14]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP9]], %[[BB50]] ]
-; CHECK-NEXT: [[TMP64]] = phi float [ [[TMP17]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ [[TMP55]], %[[BB53]] ], [ [[TMP10]], %[[BB50]] ]
-; CHECK-NEXT: [[TMP65]] = phi float [ [[TMP16]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP11]], %[[BB50]] ]
-; CHECK-NEXT: [[TMP66]] = phi float [ [[TMP15]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ 0.000000e+00, %[[BB50]] ]
-; CHECK-NEXT: [[TMP67]] = phi float [ [[TMP18]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ 0.000000e+00, %[[BB50]] ]
-; CHECK-NEXT: [[TMP68]] = phi float [ [[TMP19]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP3]], %[[BB50]] ]
-; CHECK-NEXT: [[TMP69]] = phi float [ [[TMP20]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ [[TMP54]], %[[BB53]] ], [ 0.000000e+00, %[[BB50]] ]
-; CHECK-NEXT: [[TMP70]] = phi float [ [[TMP21]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP51]], %[[BB50]] ]
-; CHECK-NEXT: [[TMP71]] = phi float [ [[TMP22]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ [[TMP54]], %[[BB53]] ], [ [[TMP52]], %[[BB50]] ]
+; CHECK-NEXT: [[TMP57]] = phi float [ [[TMP9]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP3]], %[[BB50]] ]
+; CHECK-NEXT: [[TMP58]] = phi float [ [[TMP10]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ [[TMP55]], %[[BB53]] ], [ 0.000000e+00, %[[BB50]] ]
+; CHECK-NEXT: [[TMP59]] = phi float [ [[TMP11]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP3]], %[[BB50]] ]
+; CHECK-NEXT: [[TMP60]] = phi float [ [[TMP12]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ 0.000000e+00, %[[BB50]] ]
+; CHECK-NEXT: [[TMP61]] = phi float [ [[TMP13]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ [[TMP54]], %[[BB53]] ], [ 0.000000e+00, %[[BB50]] ]
+; CHECK-NEXT: [[TMP62]] = phi float [ [[TMP14]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP9]], %[[BB50]] ]
+; CHECK-NEXT: [[TMP63]] = phi float [ [[TMP15]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ [[TMP55]], %[[BB53]] ], [ [[TMP10]], %[[BB50]] ]
+; CHECK-NEXT: [[TMP64]] = phi float [ [[TMP16]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP11]], %[[BB50]] ]
+; CHECK-NEXT: [[TMP65]] = phi float [ [[TMP17]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ 0.000000e+00, %[[BB50]] ]
+; CHECK-NEXT: [[TMP66]] = phi float [ [[TMP18]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ 0.000000e+00, %[[BB50]] ]
+; CHECK-NEXT: [[TMP67]] = phi float [ [[TMP19]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP3]], %[[BB50]] ]
+; CHECK-NEXT: [[TMP68]] = phi float [ [[TMP20]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ [[TMP54]], %[[BB53]] ], [ 0.000000e+00, %[[BB50]] ]
+; CHECK-NEXT: [[TMP69]] = phi float [ [[TMP21]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP50]], %[[BB50]] ]
+; CHECK-NEXT: [[TMP70]] = phi float [ [[TMP22]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ [[TMP54]], %[[BB53]] ], [ [[TMP51]], %[[BB50]] ]
; CHECK-NEXT: [[TMP72]] = phi float [ [[TMP24]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP24]], %[[BB50]] ]
; CHECK-NEXT: [[TMP73]] = phi float [ [[TMP23]], %[[BB29]] ], [ 0.000000e+00, %[[BB27]] ], [ 0.000000e+00, %[[BB8]] ], [ 0.000000e+00, %[[BB53]] ], [ [[TMP23]], %[[BB50]] ]
; CHECK-NEXT: [[TMP74:%.*]] = phi <4 x float> [ [[TMP25]], %[[BB29]] ], [ [[TMP25]], %[[BB27]] ], [ zeroinitializer, %[[BB8]] ], [ [[TMP34]], %[[BB53]] ], [ [[TMP34]], %[[BB50]] ]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/scalarize-ctlz.ll b/llvm/test/Transforms/SLPVectorizer/X86/scalarize-ctlz.ll
index 0ec62450fac64..b9815a3c2e970 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/scalarize-ctlz.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/scalarize-ctlz.ll
@@ -95,25 +95,21 @@ define <4 x i64> @scalarize_ctlz_v4i64(<4 x i64> %v) {
define <8 x i64> @scalarize_ctlz_v8i64(<8 x i64> %v) {
; SSE2-LABEL: define <8 x i64> @scalarize_ctlz_v8i64(
; SSE2-SAME: <8 x i64> [[V:%.*]]) #[[ATTR0]] {
-; SSE2-NEXT: [[V2:%.*]] = extractelement <8 x i64> [[V]], i64 2
-; SSE2-NEXT: [[V3:%.*]] = extractelement <8 x i64> [[V]], i64 3
-; SSE2-NEXT: [[V4:%.*]] = extractelement <8 x i64> [[V]], i64 4
-; SSE2-NEXT: [[V5:%.*]] = extractelement <8 x i64> [[V]], i64 5
; SSE2-NEXT: [[V6:%.*]] = extractelement <8 x i64> [[V]], i64 6
; SSE2-NEXT: [[V7:%.*]] = extractelement <8 x i64> [[V]], i64 7
; SSE2-NEXT: [[TMP1:%.*]] = shufflevector <8 x i64> [[V]], <8 x i64> poison, <2 x i32> <i32 0, i32 1>
; SSE2-NEXT: [[TMP8:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[TMP1]], i1 false)
-; SSE2-NEXT: [[C2:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V2]], i1 false)
-; SSE2-NEXT: [[C3:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V3]], i1 false)
-; SSE2-NEXT: [[C4:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V4]], i1 false)
-; SSE2-NEXT: [[C5:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V5]], i1 false)
+; SSE2-NEXT: [[TMP3:%.*]] = shufflevector <8 x i64> [[V]], <8 x i64> poison, <2 x i32> <i32 2, i32 3>
+; SSE2-NEXT: [[TMP4:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[TMP3]], i1 false)
+; SSE2-NEXT: [[TMP5:%.*]] = shufflevector <8 x i64> [[V]], <8 x i64> poison, <2 x i32> <i32 4, i32 5>
+; SSE2-NEXT: [[TMP6:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[TMP5]], i1 false)
; SSE2-NEXT: [[C6:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V6]], i1 false)
; SSE2-NEXT: [[C7:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V7]], i1 false)
; SSE2-NEXT: [[TMP12:%.*]] = shufflevector <2 x i64> [[TMP8]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE2-NEXT: [[R2:%.*]] = insertelement <8 x i64> [[TMP12]], i64 [[C2]], i64 2
-; SSE2-NEXT: [[R3:%.*]] = insertelement <8 x i64> [[R2]], i64 [[C3]], i64 3
-; SSE2-NEXT: [[R4:%.*]] = insertelement <8 x i64> [[R3]], i64 [[C4]], i64 4
-; SSE2-NEXT: [[R5:%.*]] = insertelement <8 x i64> [[R4]], i64 [[C5]], i64 5
+; SSE2-NEXT: [[TMP11:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT: [[TMP9:%.*]] = shufflevector <2 x i64> [[TMP8]], <2 x i64> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT: [[TMP10:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT: [[R5:%.*]] = shufflevector <8 x i64> [[TMP9]], <8 x i64> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 poison, i32 poison>
; SSE2-NEXT: [[R6:%.*]] = insertelement <8 x i64> [[R5]], i64 [[C6]], i64 6
; SSE2-NEXT: [[R73:%.*]] = insertelement <8 x i64> [[R6]], i64 [[C7]], i64 7
; SSE2-NEXT: ret <8 x i64> [[R73]]
diff --git a/llvm/test/Transforms/SLPVectorizer/alternate-non-profitable.ll b/llvm/test/Transforms/SLPVectorizer/alternate-non-profitable.ll
index 97628b26e65e5..650187bd4ec43 100644
--- a/llvm/test/Transforms/SLPVectorizer/alternate-non-profitable.ll
+++ b/llvm/test/Transforms/SLPVectorizer/alternate-non-profitable.ll
@@ -76,9 +76,14 @@ define <2 x float> @replace_through_casts_and_binop_and_unop(i16 %inp) {
; CHECK-LABEL: define <2 x float> @replace_through_casts_and_binop_and_unop(
; CHECK-SAME: i16 [[INP:%.*]]) {
; CHECK-NEXT: [[ADD:%.*]] = add nsw i16 [[INP]], -10
-; CHECK-NEXT: [[TMP1:%.*]] = sitofp i16 [[ADD]] to float
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i16> poison, i16 [[ADD]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i16> [[TMP6]], <2 x i16> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = sitofp <2 x i16> [[TMP7]] to <2 x float>
+; CHECK-NEXT: [[TMP9:%.*]] = uitofp <2 x i16> [[TMP7]] to <2 x float>
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x float> [[TMP8]], <2 x float> [[TMP9]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x float> [[TMP10]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = fneg float [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = uitofp i16 [[ADD]] to float
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP10]], i64 1
; CHECK-NEXT: [[TMP4:%.*]] = fadd float [[TMP3]], 2.000000e+00
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x float> poison, float [[TMP4]], i64 0
; CHECK-NEXT: [[R:%.*]] = insertelement <2 x float> [[TMP5]], float [[TMP2]], i64 1
@@ -98,9 +103,14 @@ define <2 x float> @replace_through_casts_through_splat(i16 %inp) {
; CHECK-LABEL: define <2 x float> @replace_through_casts_through_splat(
; CHECK-SAME: i16 [[INP:%.*]]) {
; CHECK-NEXT: [[ADD:%.*]] = add nsw i16 [[INP]], -10
-; CHECK-NEXT: [[TMP1:%.*]] = uitofp i16 [[ADD]] to float
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i16> poison, i16 [[ADD]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i16> [[TMP6]], <2 x i16> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = uitofp <2 x i16> [[TMP7]] to <2 x float>
+; CHECK-NEXT: [[TMP9:%.*]] = sitofp <2 x i16> [[TMP7]] to <2 x float>
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x float> [[TMP8]], <2 x float> [[TMP9]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x float> [[TMP10]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = fadd float [[TMP1]], 2.000000e+00
-; CHECK-NEXT: [[TMP3:%.*]] = sitofp i16 [[ADD]] to float
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP10]], i64 1
; CHECK-NEXT: [[TMP4:%.*]] = fneg float [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x float> poison, float [[TMP2]], i64 0
; CHECK-NEXT: [[R:%.*]] = insertelement <2 x float> [[TMP5]], float [[TMP4]], i64 1
diff --git a/llvm/test/Transforms/SLPVectorizer/insert-element-build-vector-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/insert-element-build-vector-inseltpoison.ll
index bf35b3fb0c07f..4efb24da88c6b 100644
--- a/llvm/test/Transforms/SLPVectorizer/insert-element-build-vector-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/insert-element-build-vector-inseltpoison.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: %if x86-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=-10000 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s %}
-; RUN: %if x86-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=0 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s %}
-; RUN: %if x86-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=-10000 -slp-min-tree-size=0 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s %}
-; RUN: %if aarch64-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=-10000 -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s %}
-; RUN: %if aarch64-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=0 -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s %}
-; RUN: %if aarch64-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=-10000 -slp-min-tree-size=0 -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s %}
+; RUN: %if x86-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=-10000 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=CHECK,FORCED %}
+; RUN: %if x86-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=0 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=CHECK,COST %}
+; RUN: %if x86-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=-10000 -slp-min-tree-size=0 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=CHECK,FORCED %}
+; RUN: %if aarch64-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=-10000 -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,FORCED %}
+; RUN: %if aarch64-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=0 -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,COST %}
+; RUN: %if aarch64-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=-10000 -slp-min-tree-size=0 -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,FORCED %}
define <4 x float> @simple_select(<4 x float> %a, <4 x float> %b, <4 x i32> %c) {
; CHECK-LABEL: @simple_select(
@@ -239,10 +239,10 @@ define <4 x float> @simple_select_no_users(<4 x float> %a, <4 x float> %b, <4 x
; to do this backwards this backwards
define <4 x i32> @reconstruct(<4 x i32> %c) {
; CHECK-LABEL: @reconstruct(
-; CHECK-NEXT: [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
-; CHECK-NEXT: [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
-; CHECK-NEXT: [[C2:%.*]] = extractelement <4 x i32> [[C]], i32 2
-; CHECK-NEXT: [[C3:%.*]] = extractelement <4 x i32> [[C]], i32 3
+; CHECK-NEXT: [[C3:%.*]] = extractelement <4 x i32> [[C:%.*]], i64 3
+; CHECK-NEXT: [[C2:%.*]] = extractelement <4 x i32> [[C]], i64 2
+; CHECK-NEXT: [[C1:%.*]] = extractelement <4 x i32> [[C]], i64 1
+; CHECK-NEXT: [[C0:%.*]] = extractelement <4 x i32> [[C]], i64 0
; CHECK-NEXT: [[RA:%.*]] = insertelement <4 x i32> poison, i32 [[C0]], i32 0
; CHECK-NEXT: [[RB:%.*]] = insertelement <4 x i32> [[RA]], i32 [[C1]], i32 1
; CHECK-NEXT: [[RC:%.*]] = insertelement <4 x i32> [[RB]], i32 [[C2]], i32 2
@@ -285,26 +285,44 @@ define <2 x float> @simple_select_v2(<2 x float> %a, <2 x float> %b, <2 x i32> %
; re-visiting the insertelement chains starting with undef
; (low cost threshold needed to force this to happen)
define <4 x float> @simple_select_partial_vector(<4 x float> %a, <4 x float> %b, <4 x i32> %c) {
-; CHECK-LABEL: @simple_select_partial_vector(
-; CHECK-NEXT: [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
-; CHECK-NEXT: [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
-; CHECK-NEXT: [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
-; CHECK-NEXT: [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
-; CHECK-NEXT: [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
-; CHECK-NEXT: [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[C0]], i32 0
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[C1]], i32 1
-; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <2 x i32> [[TMP2]], zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x float> poison, float [[A0]], i32 0
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x float> [[TMP4]], float [[A1]], i32 1
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x float> poison, float [[B0]], i32 0
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[B1]], i32 1
-; CHECK-NEXT: [[TMP8:%.*]] = select <2 x i1> [[TMP3]], <2 x float> [[TMP5]], <2 x float> [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x float> [[TMP8]], i32 0
-; CHECK-NEXT: [[RA:%.*]] = insertelement <4 x float> poison, float [[TMP9]], i32 0
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x float> [[TMP8]], i32 1
-; CHECK-NEXT: [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP10]], i32 1
-; CHECK-NEXT: ret <4 x float> [[RB]]
+; FORCED-LABEL: @simple_select_partial_vector(
+; FORCED-NEXT: [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
+; FORCED-NEXT: [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
+; FORCED-NEXT: [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; FORCED-NEXT: [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
+; FORCED-NEXT: [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; FORCED-NEXT: [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
+; FORCED-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[C0]], i32 0
+; FORCED-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[C1]], i32 1
+; FORCED-NEXT: [[TMP3:%.*]] = icmp ne <2 x i32> [[TMP2]], zeroinitializer
+; FORCED-NEXT: [[TMP4:%.*]] = insertelement <2 x float> poison, float [[A0]], i32 0
+; FORCED-NEXT: [[TMP5:%.*]] = insertelement <2 x float> [[TMP4]], float [[A1]], i32 1
+; FORCED-NEXT: [[TMP6:%.*]] = insertelement <2 x float> poison, float [[B0]], i32 0
+; FORCED-NEXT: [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[B1]], i32 1
+; FORCED-NEXT: [[TMP8:%.*]] = select <2 x i1> [[TMP3]], <2 x float> [[TMP5]], <2 x float> [[TMP7]]
+; FORCED-NEXT: [[TMP9:%.*]] = extractelement <2 x float> [[TMP8]], i32 0
+; FORCED-NEXT: [[RA:%.*]] = insertelement <4 x float> poison, float [[TMP9]], i32 0
+; FORCED-NEXT: [[TMP10:%.*]] = extractelement <2 x float> [[TMP8]], i32 1
+; FORCED-NEXT: [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP10]], i32 1
+; FORCED-NEXT: ret <4 x float> [[RB]]
+;
+; COST-LABEL: @simple_select_partial_vector(
+; COST-NEXT: [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
+; COST-NEXT: [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
+; COST-NEXT: [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; COST-NEXT: [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
+; COST-NEXT: [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; COST-NEXT: [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
+; COST-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[C0]], i32 0
+; COST-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[C1]], i32 1
+; COST-NEXT: [[TMP3:%.*]] = icmp ne <2 x i32> [[TMP2]], zeroinitializer
+; COST-NEXT: [[TMP4:%.*]] = insertelement <2 x float> poison, float [[A0]], i32 0
+; COST-NEXT: [[TMP5:%.*]] = insertelement <2 x float> [[TMP4]], float [[A1]], i32 1
+; COST-NEXT: [[TMP6:%.*]] = insertelement <2 x float> poison, float [[B0]], i32 0
+; COST-NEXT: [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[B1]], i32 1
+; COST-NEXT: [[TMP8:%.*]] = select <2 x i1> [[TMP3]], <2 x float> [[TMP5]], <2 x float> [[TMP7]]
+; COST-NEXT: [[TMP9:%.*]] = shufflevector <2 x float> [[TMP8]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; COST-NEXT: ret <4 x float> [[TMP9]]
;
%c0 = extractelement <4 x i32> %c, i32 0
%c1 = extractelement <4 x i32> %c, i32 1
diff --git a/llvm/test/Transforms/SLPVectorizer/insert-element-build-vector.ll b/llvm/test/Transforms/SLPVectorizer/insert-element-build-vector.ll
index ee07f451c5a1d..c3759095d544f 100644
--- a/llvm/test/Transforms/SLPVectorizer/insert-element-build-vector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/insert-element-build-vector.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: %if x86-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=-10000 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s %}
-; RUN: %if x86-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=0 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s %}
-; RUN: %if x86-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=-10000 -slp-min-tree-size=0 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s %}
-; RUN: %if aarch64-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=-10000 -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s %}
-; RUN: %if aarch64-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=0 -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s %}
-; RUN: %if aarch64-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=-10000 -slp-min-tree-size=0 -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s %}
+; RUN: %if x86-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=-10000 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=CHECK,FORCED %}
+; RUN: %if x86-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=0 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=CHECK,COST %}
+; RUN: %if x86-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=-10000 -slp-min-tree-size=0 -mtriple=x86_64-unknown-unknown < %s | FileCheck %s --check-prefixes=CHECK,FORCED %}
+; RUN: %if aarch64-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=-10000 -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,FORCED %}
+; RUN: %if aarch64-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=0 -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,COST %}
+; RUN: %if aarch64-registered-target %{ opt -S -passes=slp-vectorizer -slp-threshold=-10000 -slp-min-tree-size=0 -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,FORCED %}
define <4 x float> @simple_select(<4 x float> %a, <4 x float> %b, <4 x i32> %c) {
; CHECK-LABEL: @simple_select(
@@ -275,10 +275,10 @@ define <4 x float> @simple_select_no_users(<4 x float> %a, <4 x float> %b, <4 x
; to do this backwards this backwards
define <4 x i32> @reconstruct(<4 x i32> %c) {
; CHECK-LABEL: @reconstruct(
-; CHECK-NEXT: [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
-; CHECK-NEXT: [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
-; CHECK-NEXT: [[C2:%.*]] = extractelement <4 x i32> [[C]], i32 2
-; CHECK-NEXT: [[C3:%.*]] = extractelement <4 x i32> [[C]], i32 3
+; CHECK-NEXT: [[C3:%.*]] = extractelement <4 x i32> [[C:%.*]], i64 3
+; CHECK-NEXT: [[C2:%.*]] = extractelement <4 x i32> [[C]], i64 2
+; CHECK-NEXT: [[C1:%.*]] = extractelement <4 x i32> [[C]], i64 1
+; CHECK-NEXT: [[C0:%.*]] = extractelement <4 x i32> [[C]], i64 0
; CHECK-NEXT: [[RA:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[C0]], i32 0
; CHECK-NEXT: [[RB:%.*]] = insertelement <4 x i32> [[RA]], i32 [[C1]], i32 1
; CHECK-NEXT: [[RC:%.*]] = insertelement <4 x i32> [[RB]], i32 [[C2]], i32 2
@@ -321,26 +321,45 @@ define <2 x float> @simple_select_v2(<2 x float> %a, <2 x float> %b, <2 x i32> %
; re-visiting the insertelement chains starting with zeroinitializer
; (low cost threshold needed to force this to happen)
define <4 x float> @simple_select_partial_vector(<4 x float> %a, <4 x float> %b, <4 x i32> %c) {
-; CHECK-LABEL: @simple_select_partial_vector(
-; CHECK-NEXT: [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
-; CHECK-NEXT: [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
-; CHECK-NEXT: [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
-; CHECK-NEXT: [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
-; CHECK-NEXT: [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
-; CHECK-NEXT: [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[C0]], i32 0
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[C1]], i32 1
-; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <2 x i32> [[TMP2]], zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x float> zeroinitializer, float [[A0]], i32 0
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x float> [[TMP4]], float [[A1]], i32 1
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x float> zeroinitializer, float [[B0]], i32 0
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[B1]], i32 1
-; CHECK-NEXT: [[TMP8:%.*]] = select <2 x i1> [[TMP3]], <2 x float> [[TMP5]], <2 x float> [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x float> [[TMP8]], i32 0
-; CHECK-NEXT: [[RA:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP9]], i32 0
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x float> [[TMP8]], i32 1
-; CHECK-NEXT: [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP10]], i32 1
-; CHECK-NEXT: ret <4 x float> [[RB]]
+; FORCED-LABEL: @simple_select_partial_vector(
+; FORCED-NEXT: [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
+; FORCED-NEXT: [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
+; FORCED-NEXT: [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; FORCED-NEXT: [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
+; FORCED-NEXT: [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; FORCED-NEXT: [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
+; FORCED-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[C0]], i32 0
+; FORCED-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[C1]], i32 1
+; FORCED-NEXT: [[TMP3:%.*]] = icmp ne <2 x i32> [[TMP2]], zeroinitializer
+; FORCED-NEXT: [[TMP4:%.*]] = insertelement <2 x float> zeroinitializer, float [[A0]], i32 0
+; FORCED-NEXT: [[TMP5:%.*]] = insertelement <2 x float> [[TMP4]], float [[A1]], i32 1
+; FORCED-NEXT: [[TMP6:%.*]] = insertelement <2 x float> zeroinitializer, float [[B0]], i32 0
+; FORCED-NEXT: [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[B1]], i32 1
+; FORCED-NEXT: [[TMP8:%.*]] = select <2 x i1> [[TMP3]], <2 x float> [[TMP5]], <2 x float> [[TMP7]]
+; FORCED-NEXT: [[TMP9:%.*]] = extractelement <2 x float> [[TMP8]], i32 0
+; FORCED-NEXT: [[RA:%.*]] = insertelement <4 x float> zeroinitializer, float [[TMP9]], i32 0
+; FORCED-NEXT: [[TMP10:%.*]] = extractelement <2 x float> [[TMP8]], i32 1
+; FORCED-NEXT: [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP10]], i32 1
+; FORCED-NEXT: ret <4 x float> [[RB]]
+;
+; COST-LABEL: @simple_select_partial_vector(
+; COST-NEXT: [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
+; COST-NEXT: [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
+; COST-NEXT: [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; COST-NEXT: [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
+; COST-NEXT: [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; COST-NEXT: [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
+; COST-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[C0]], i32 0
+; COST-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[C1]], i32 1
+; COST-NEXT: [[TMP3:%.*]] = icmp ne <2 x i32> [[TMP2]], zeroinitializer
+; COST-NEXT: [[TMP4:%.*]] = insertelement <2 x float> zeroinitializer, float [[A0]], i32 0
+; COST-NEXT: [[TMP5:%.*]] = insertelement <2 x float> [[TMP4]], float [[A1]], i32 1
+; COST-NEXT: [[TMP6:%.*]] = insertelement <2 x float> zeroinitializer, float [[B0]], i32 0
+; COST-NEXT: [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[B1]], i32 1
+; COST-NEXT: [[TMP8:%.*]] = select <2 x i1> [[TMP3]], <2 x float> [[TMP5]], <2 x float> [[TMP7]]
+; COST-NEXT: [[TMP9:%.*]] = shufflevector <2 x float> [[TMP8]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; COST-NEXT: [[TMP10:%.*]] = shufflevector <4 x float> zeroinitializer, <4 x float> [[TMP9]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; COST-NEXT: ret <4 x float> [[TMP10]]
;
%c0 = extractelement <4 x i32> %c, i32 0
%c1 = extractelement <4 x i32> %c, i32 1
diff --git a/llvm/test/Transforms/SLPVectorizer/minbitwidth-multiuse-with-insertelement.ll b/llvm/test/Transforms/SLPVectorizer/minbitwidth-multiuse-with-insertelement.ll
index 3351afab47207..29d964b068801 100644
--- a/llvm/test/Transforms/SLPVectorizer/minbitwidth-multiuse-with-insertelement.ll
+++ b/llvm/test/Transforms/SLPVectorizer/minbitwidth-multiuse-with-insertelement.ll
@@ -9,10 +9,11 @@ define void @test(i8 %0) {
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> <i8 0, i8 poison>, i8 [[TMP0]], i64 1
; CHECK-NEXT: [[TMP2:%.*]] = sext <2 x i8> [[TMP1]] to <2 x i32>
; CHECK-NEXT: [[TMP3:%.*]] = mul <2 x i8> [[TMP1]], zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i8> [[TMP3]], i64 0
-; CHECK-NEXT: [[TMP5:%.*]] = zext i8 [[TMP4]] to i32
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i8> [[TMP3]], i64 1
-; CHECK-NEXT: [[TMP7:%.*]] = zext i8 [[TMP6]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = trunc <2 x i8> [[TMP3]] to <2 x i1>
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = zext i1 [[TMP6]] to i32
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP4]], i64 1
+; CHECK-NEXT: [[TMP7:%.*]] = zext i1 [[TMP9]] to i32
; CHECK-NEXT: [[ADD:%.*]] = or i32 [[TMP5]], [[TMP7]]
; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[ADD]], 1
; CHECK-NEXT: [[CONV9:%.*]] = trunc i32 [[SHR]] to i8
More information about the llvm-commits
mailing list