[llvm] [SLP] Refine loop-aware gather cost and admit sibling-loop subtrees (PR #192801)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Sat Apr 18 12:51:18 PDT 2026
https://github.com/alexey-bataev updated https://github.com/llvm/llvm-project/pull/192801
>From 769d8d27fce58f15347152f31f07b53a40eed59f Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sat, 18 Apr 2026 10:35:32 -0700
Subject: [PATCH 1/2] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20in?=
=?UTF-8?q?itial=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 221 +++++++++++++----
.../AArch64/externally-used-copyables.ll | 224 ++++++++----------
.../SLPVectorizer/AArch64/getelementptr.ll | 11 +-
.../SLPVectorizer/AArch64/slp-fma-loss.ll | 19 +-
.../SystemZ/vec-elt-insertion.ll | 2 +-
.../X86/buildvectors-parent-phi-nodes.ll | 12 +-
llvm/test/Transforms/SLPVectorizer/X86/cse.ll | 7 +-
.../X86/deleted-instructions-clear.ll | 17 +-
.../deleted-node-with-copyable-operands.ll | 26 +-
...t-node-schedulable-with-multi-copyables.ll | 54 ++---
.../X86/parent-node-split-non-schedulable.ll | 43 ++--
...duced-val-extracted-and-externally-used.ll | 35 +--
12 files changed, 378 insertions(+), 293 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 6625384616c26..41b1464f07f94 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -243,6 +243,18 @@ static cl::opt<unsigned> LoopAwareTripCount(
cl::desc("Loop trip count, considered by the cost model during "
"modeling (0=loops are ignored and considered flat code)"));
+/// Refine the loop-aware cost scaling of gather/buildvector tree entries by
+/// using the per-lane execution scale of the operand that feeds each lane,
+/// instead of a single whole-entry scale. This matches the LICM hoisting
+/// performed by optimizeGatherSequence() at codegen time: lanes whose
+/// operands are loop-invariant in an inner loop contribute the outer loop's
+/// execution scale rather than the inner loop's, which avoids over-costing
+/// buildvectors that bridge values from outer loop nests into an inner loop.
+static cl::opt<bool> PerLaneGatherScale(
+ "slp-per-lane-gather-scale", cl::init(true), cl::Hidden,
+ cl::desc("Use per-lane execution scale for gather/buildvector tree "
+ "entries to model LICM-hoistable buildvector sequences."));
+
// Limit the number of alias checks. The limit is chosen so that
// it has no negative effect on the llvm benchmarks.
static const unsigned AliasedCheckLimit = 10;
@@ -3890,10 +3902,27 @@ class slpvectorizer::BoUpSLP {
/// external use.
/// \p U is the user of the vectorized value from the entry, if using the
/// parent for the external use.
- unsigned getScaleToLoopIterations(const TreeEntry &TE,
+ uint64_t getScaleToLoopIterations(const TreeEntry &TE,
Value *Scalar = nullptr,
Instruction *U = nullptr);
+ /// \returns the product of trip counts of the loop \p L and all of its
+ /// enclosing loops. Unlike the state kept by getScaleToLoopIterations(),
+ /// this helper depends only on the loop structure and is independent of
+ /// per-entry operand invariance. Returns 1 when loop-aware cost modeling
+ /// is disabled or \p L is null.
+ uint64_t getLoopNestScale(const Loop *L);
+
+ /// \returns a refined execution scale for a gather/buildvector tree entry
+ /// \p TE. The scale is computed as the average of per-lane execution
+ /// scales: each lane's scale is the loop-nest scale of the loop that
+ /// contains the lane's defining instruction (or 1 if the lane is a
+ /// constant / loop-invariant non-instruction value). This models the
+ /// LICM hoisting that optimizeGatherSequence() performs after vectorization
+ /// for inserts with loop-invariant operands. Falls back to the whole-entry
+ /// scale when per-lane information is unavailable or the feature is off.
+ uint64_t getGatherNodeEffectiveScale(const TreeEntry &TE);
+
/// Get the loop nest for the given loop \p L.
ArrayRef<const Loop *> getLoopNest(const Loop *L);
@@ -4867,9 +4896,10 @@ class slpvectorizer::BoUpSLP {
/// Maps the loops to their loop nests.
SmallDenseMap<const Loop *, SmallVector<const Loop *>> LoopToLoopNest;
- /// Maps the loops to their scale factor, which is built as a multiplication
- /// of the tripcounts of the loops in the loop nest.
- SmallDenseMap<const Loop *, unsigned> LoopToScaleFactor;
+ /// Per-loop cache of nest scale factors: the product of trip counts of the
+ /// loop and all of its ancestors. Shared by getLoopNestScale() and (via it)
+ /// by getScaleToLoopIterations() and getGatherNodeEffectiveScale().
+ SmallDenseMap<const Loop *, uint64_t> LoopNestScaleCache;
/// This POD struct describes one external user in the vectorized tree.
struct ExternalUser {
@@ -12431,21 +12461,61 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
S.getMainOp()->getParent()) {
BasicBlock *Parent = S.getMainOp()->getParent();
if (const Loop *L = LI->getLoopFor(Parent)) {
- // Check that the new loop nest is not involved.
- // Otherwise, mark it as a gather node.
+ // Check that the new loop nest shares the same outer structure as the
+ // tree's current loop nest. Completely disjoint nests (different
+ // outermost loops) are forced to gather because their scales cannot be
+ // meaningfully combined. Sibling inner loops inside a common outer
+ // loop are allowed: the cost model scales each entry by its own loop
+ // via getScaleToLoopIterations(), so a tree that spans sibling inner
+ // loops (e.g. a PHI at their merge block) can still be costed
+ // correctly. Contract CurrentLoopNest to the longest common prefix
+ // with the new entry's nest so subsequent entries in yet another
+ // sibling can also be admitted.
L = findInnermostNonInvariantLoop(L, VL);
if (L) {
SmallVector<const Loop *> NewLoopNest(getLoopNest(L));
+ unsigned CommonLen = 0;
for (const auto [L1, L2] : zip(CurrentLoopNest, NewLoopNest)) {
- if (L1 != L2) {
- LLVM_DEBUG(dbgs() << "SLP: Different loop nest.\n");
+ if (L1 != L2)
+ break;
+ ++CommonLen;
+ }
+ if (!CurrentLoopNest.empty() && CommonLen == 0) {
+ // No shared outer loop at all.
+ LLVM_DEBUG(dbgs() << "SLP: Different loop nest.\n");
+ newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
+ return;
+ }
+ if (CurrentLoopNest.empty()) {
+ CurrentLoopNest.assign(NewLoopNest);
+ } else if (CommonLen < CurrentLoopNest.size() &&
+ CommonLen < NewLoopNest.size()) {
+ // Divergence below the common prefix: the tree now spans sibling
+ // loops at depth CommonLen. Admitting them into one tree makes
+ // the profitability decision JOINT across both siblings, so a
+ // very hot sibling could otherwise let an unprofitable cold
+ // sibling ride along "for free" (per-entry scaling of the cold
+ // sibling's entries would be dwarfed by the hot one). Require
+ // SCEV-proven equal backedge-taken counts for the diverging
+ // siblings before joining; otherwise force gather.
+ const Loop *SibA = CurrentLoopNest[CommonLen];
+ const Loop *SibB = NewLoopNest[CommonLen];
+ const SCEV *BecA = SE->getBackedgeTakenCount(SibA);
+ const SCEV *BecB = SE->getBackedgeTakenCount(SibB);
+ if (isa<SCEVCouldNotCompute>(BecA) || BecA != BecB) {
+ LLVM_DEBUG(dbgs()
+ << "SLP: Sibling loops have different trip counts.\n");
newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
return;
}
+ CurrentLoopNest.truncate(CommonLen);
+ } else if (NewLoopNest.size() > CurrentLoopNest.size()) {
+ // New entry lives deeper in the same nest chain; extend.
+ CurrentLoopNest.append(
+ std::next(NewLoopNest.begin(), CurrentLoopNest.size()),
+ NewLoopNest.end());
}
- if (NewLoopNest.size() > CurrentLoopNest.size())
- CurrentLoopNest.append(std::next(NewLoopNest.begin(), CurrentLoopNest.size()),
- NewLoopNest.end());
+ // Otherwise NewLoopNest is a prefix of CurrentLoopNest: keep as-is.
}
}
}
@@ -15841,7 +15911,7 @@ static unsigned getLoopTripCount(const Loop *L, ScalarEvolution &SE) {
return LoopAwareTripCount;
}
-unsigned BoUpSLP::getScaleToLoopIterations(const TreeEntry &TE, Value *Scalar,
+uint64_t BoUpSLP::getScaleToLoopIterations(const TreeEntry &TE, Value *Scalar,
Instruction *U) {
BasicBlock *Parent = nullptr;
if (U) {
@@ -15868,31 +15938,77 @@ unsigned BoUpSLP::getScaleToLoopIterations(const TreeEntry &TE, Value *Scalar,
} else {
Parent = TE.getMainOp()->getParent();
}
- if (const Loop *L = LI->getLoopFor(Parent)) {
- const auto It = LoopToScaleFactor.find(L);
- if (It != LoopToScaleFactor.end())
- return It->second;
- unsigned Scale = 1;
- if (const Loop *NonInvL = findInnermostNonInvariantLoop(
- L, Scalar ? ArrayRef(Scalar) : ArrayRef(TE.Scalars))) {
- Scale = getLoopTripCount(NonInvL, *SE);
- for (const Loop *LN : getLoopNest(NonInvL)) {
- if (LN == L)
- break;
- auto LNRes = LoopToScaleFactor.try_emplace(LN, 0);
- auto &LoopScale = LNRes.first->getSecond();
- if (!LNRes.second) {
- Scale *= LoopScale;
- break;
- }
- Scale *= getLoopTripCount(LN, *SE);
- LoopScale = Scale;
- }
- }
- LoopToScaleFactor.try_emplace(L, Scale);
- return Scale;
- }
- return 1;
+ const Loop *L = LI->getLoopFor(Parent);
+ if (!L)
+ return 1;
+ // The entry's cost is paid once per execution of the innermost loop in
+ // which some of its operands are variant. Operands that are invariant in
+ // all enclosing loops are executed once (LICM will hoist them out).
+ return getLoopNestScale(findInnermostNonInvariantLoop(
+ L, Scalar ? ArrayRef(Scalar) : ArrayRef(TE.Scalars)));
+}
+
+uint64_t BoUpSLP::getLoopNestScale(const Loop *L) {
+ if (!L || LoopAwareTripCount == 0)
+ return 1;
+ if (auto It = LoopNestScaleCache.find(L); It != LoopNestScaleCache.end())
+ return It->second;
+ // Collect loops from L outward up to (but not including) the first cached
+ // ancestor or the function top, then walk back inward multiplying trip
+ // counts. Use uint64_t to avoid silent overflow on deep/large nests.
+ SmallVector<const Loop *> Chain;
+ for (const Loop *Cur = L; Cur; Cur = Cur->getParentLoop()) {
+ if (LoopNestScaleCache.contains(Cur))
+ break;
+ Chain.push_back(Cur);
+ }
+ assert(!Chain.empty() && "Early-return above should have handled cache hit.");
+ uint64_t Scale = 1;
+ if (const Loop *Parent = Chain.back()->getParentLoop())
+ Scale = LoopNestScaleCache.lookup(Parent);
+ // Walk from the outermost uncached loop inward, accumulating trip counts.
+ // Use SaturatingMultiply to clamp at uint64_t max on deep/large nests
+ // rather than wrapping around.
+ for (const Loop *Cur : reverse(Chain)) {
+ uint64_t TC = std::max<uint64_t>(1, getLoopTripCount(Cur, *SE));
+ Scale = SaturatingMultiply(Scale, TC);
+ LoopNestScaleCache.try_emplace(Cur, std::max<uint64_t>(1, Scale));
+ }
+ return std::max<uint64_t>(1, Scale);
+}
+
+uint64_t BoUpSLP::getGatherNodeEffectiveScale(const TreeEntry &TE) {
+ // Only meaningful for gather/buildvector-like entries; the per-lane
+ // insertelements that make up such an entry are LICM-hoistable by
+ // optimizeGatherSequence() when their operand is loop-invariant.
+ assert((TE.isGather() || TE.State == TreeEntry::SplitVectorize) &&
+ "Expected gather/split tree entry.");
+
+ uint64_t BaseScale = getScaleToLoopIterations(TE);
+ if (!PerLaneGatherScale || LoopAwareTripCount == 0 || BaseScale <= 1)
+ return BaseScale;
+
+ // Average the per-lane execution scales: for each lane, reuse the same
+ // scale helper the rest of the cost model uses, but ask it about that
+ // one lane's value. Lanes that are loop-invariant in the current nest
+ // collapse to their outer-loop scale (or 1 for fully invariant/constant
+ // lanes), which matches the LICM hoisting performed by
+ // optimizeGatherSequence(). Cap per-lane contributions by BaseScale so a
+ // refinement can never raise the cost above the whole-entry scale.
+ uint64_t Sum = 0;
+ unsigned N = 0;
+ for (Value *V : TE.Scalars) {
+ if (isConstant(V))
+ continue;
+ ++N;
+ uint64_t LaneScale = getScaleToLoopIterations(TE, V);
+ Sum += std::min(LaneScale, BaseScale);
+ }
+ if (N == 0)
+ return BaseScale;
+ // Ceil-divide so we never round the effective scale down below 1.
+ uint64_t Avg = (Sum + N - 1) / N;
+ return std::clamp<uint64_t>(Avg, 1, BaseScale);
}
InstructionCost
@@ -17641,7 +17757,7 @@ InstructionCost BoUpSLP::getSpillCost() {
if (It != MinBWs.end())
ScalarTy = IntegerType::get(ScalarTy->getContext(), It->second.first);
auto *VecTy = getWidenedType(ScalarTy, Op->getVectorFactor());
- unsigned Scale = getScaleToLoopIterations(*Op);
+ uint64_t Scale = getScaleToLoopIterations(*Op);
InstructionCost KeepLiveCost = TTI->getCostOfKeepingLiveOverCall(VecTy);
KeepLiveCost *= Scale;
Cost += KeepLiveCost;
@@ -18008,8 +18124,8 @@ InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable(
};
constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
InstructionCost Cost = 0;
- SmallDenseMap<const TreeEntry *, unsigned> EntryToScale;
- unsigned PrevScale = 0;
+ SmallDenseMap<const TreeEntry *, uint64_t> EntryToScale;
+ uint64_t PrevScale = 0;
BasicBlock *PrevVecParent = nullptr;
for (const std::unique_ptr<TreeEntry> &Ptr : VectorizableTree) {
TreeEntry &TE = *Ptr;
@@ -18044,8 +18160,14 @@ InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable(
"Expected gather nodes with users only.");
InstructionCost C = getEntryCost(&TE, VectorizedVals, CheckedExtracts);
- unsigned Scale = 0;
+ uint64_t Scale = 0;
bool CostIsFree = C == 0;
+ // For gather/buildvector (and split-vectorize) entries, prefer the
+ // per-lane refined scale that accounts for LICM-hoistable insertelements
+ // when an operand is invariant in the current loop nest but defined in
+ // an outer loop. This prevents over-costing cross-loop-nest buildvectors.
+ const bool IsGatherLike =
+ TE.isGather() || TE.State == TreeEntry::SplitVectorize;
if (!CostIsFree && !TE.isGather() && TE.hasState()) {
if (PrevVecParent == TE.getMainOp()->getParent()) {
Scale = PrevScale;
@@ -18054,7 +18176,8 @@ InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable(
}
}
if (!CostIsFree && !Scale) {
- Scale = getScaleToLoopIterations(TE);
+ Scale = IsGatherLike ? getGatherNodeEffectiveScale(TE)
+ : getScaleToLoopIterations(TE);
C *= Scale;
EntryToScale.try_emplace(&TE, Scale);
if (!TE.isGather() && TE.hasState()) {
@@ -18411,9 +18534,13 @@ InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable(
NodesCosts.try_emplace(TE.get(), C);
continue;
}
- unsigned Scale = EntryToScale.lookup(TE.get());
- if (!Scale)
- Scale = getScaleToLoopIterations(*TE.get());
+ uint64_t Scale = EntryToScale.lookup(TE.get());
+ if (!Scale) {
+ const bool IsGatherLike =
+ TE->isGather() || TE->State == TreeEntry::SplitVectorize;
+ Scale = IsGatherLike ? getGatherNodeEffectiveScale(*TE.get())
+ : getScaleToLoopIterations(*TE.get());
+ }
C *= Scale;
NodesCosts.try_emplace(TE.get(), C);
}
@@ -18491,13 +18618,13 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
}
InstructionCost Cost = TreeCost;
- SmallDenseMap<std::tuple<const TreeEntry *, Value *, Instruction *>, unsigned>
+ SmallDenseMap<std::tuple<const TreeEntry *, Value *, Instruction *>, uint64_t>
EntryToScale;
auto ScaleCost = [&](InstructionCost C, const TreeEntry &TE,
Value *Scalar = nullptr, Instruction *U = nullptr) {
if (!C.isValid() || C == 0)
return C;
- unsigned &Scale =
+ uint64_t &Scale =
EntryToScale.try_emplace(std::make_tuple(&TE, Scalar, U), 0)
.first->getSecond();
if (!Scale)
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll
index 0eeb72570635c..e11805feef2b8 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll
@@ -5,171 +5,143 @@ define void @test(i64 %0, i64 %1, i64 %2, i64 %3, i64 %.sroa.3341.0.copyload, i6
; CHECK-LABEL: define void @test(
; CHECK-SAME: i64 [[TMP0:%.*]], i64 [[TMP1:%.*]], i64 [[TMP2:%.*]], i64 [[TMP3:%.*]], i64 [[DOTSROA_3341_0_COPYLOAD:%.*]], i64 [[DOTSROA_3308_0_COPYLOAD:%.*]], i64 [[DOTNEG1:%.*]], i64 [[INDVAR3788:%.*]], i64 [[TMP4:%.*]], i64 [[TMP5:%.*]], i64 [[TMP6:%.*]], i64 [[TMP7:%.*]], i64 [[TMP8:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[_LR_PH_PREHEADER:.*:]]
-; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP0]], 24
; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP0]], 40
; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP0]], 48
-; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[TMP0]], 1
-; CHECK-NEXT: [[TMP13:%.*]] = sub i64 1, [[TMP0]]
-; CHECK-NEXT: [[TMP14:%.*]] = shl i64 [[TMP0]], 11
-; CHECK-NEXT: [[TMP15:%.*]] = sub i64 1, [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[TMP14]], 8
-; CHECK-NEXT: [[TMP17:%.*]] = or i64 [[TMP0]], 1
+; CHECK-NEXT: [[TMP35:%.*]] = insertelement <2 x i64> poison, i64 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <2 x i64> [[TMP35]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = add <2 x i64> [[TMP12]], <i64 0, i64 1>
+; CHECK-NEXT: [[TMP14:%.*]] = shl <2 x i64> [[TMP12]], <i64 11, i64 0>
+; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <2 x i64> [[TMP35]], <2 x i64> [[TMP14]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: [[TMP16:%.*]] = sub <2 x i64> splat (i64 1), [[TMP15]]
+; CHECK-NEXT: [[TMP17:%.*]] = add <2 x i64> [[TMP14]], <i64 8, i64 1>
+; CHECK-NEXT: [[TMP37:%.*]] = or <2 x i64> [[TMP14]], <i64 8, i64 1>
+; CHECK-NEXT: [[TMP40:%.*]] = shufflevector <2 x i64> [[TMP17]], <2 x i64> [[TMP37]], <2 x i32> <i32 0, i32 3>
; CHECK-NEXT: [[TMP18:%.*]] = shl i64 [[TMP0]], 1
; CHECK-NEXT: [[TMP19:%.*]] = or i64 [[TMP18]], [[TMP0]]
; CHECK-NEXT: [[TMP20:%.*]] = or i64 [[TMP19]], 1
+; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <2 x i64> [[TMP12]], <2 x i64> <i64 24, i64 poison>, <2 x i32> <i32 2, i32 0>
+; CHECK-NEXT: [[TMP41:%.*]] = mul <2 x i64> [[TMP12]], [[TMP23]]
+; CHECK-NEXT: [[TMP44:%.*]] = insertelement <32 x i64> poison, i64 [[TMP1]], i32 5
+; CHECK-NEXT: [[TMP45:%.*]] = insertelement <4 x i64> poison, i64 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <4 x i64> [[TMP45]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP46:%.*]] = shufflevector <2 x i64> [[TMP41]], <2 x i64> poison, <32 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP74:%.*]] = shufflevector <32 x i64> [[TMP44]], <32 x i64> [[TMP46]], <32 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 32, i32 poison, i32 poison, i32 poison, i32 36, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP0]], [[TMP0]]
+; CHECK-NEXT: [[TMP85:%.*]] = shufflevector <2 x i64> [[TMP15]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP32:%.*]] = shufflevector <2 x i64> [[TMP40]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP33:%.*]] = shufflevector <4 x i64> <i64 poison, i64 poison, i64 2, i64 2>, <4 x i64> [[TMP32]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
; CHECK-NEXT: br label %[[DOTLR_PH1977_US:.*]]
; CHECK: [[_LR_PH1977_US:.*:]]
; CHECK-NEXT: [[INDVAR37888:%.*]] = phi i64 [ 0, [[DOTLR_PH_PREHEADER:%.*]] ], [ 1, %[[DOTLR_PH1977_US]] ]
-; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP16]], [[TMP0]]
-; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP20]], [[TMP0]]
-; CHECK-NEXT: [[TMP24:%.*]] = mul i64 [[TMP15]], [[TMP0]]
-; CHECK-NEXT: [[TMP25:%.*]] = mul i64 [[TMP17]], [[TMP0]]
+; CHECK-NEXT: [[TMP34:%.*]] = mul <4 x i64> [[TMP27]], [[TMP33]]
+; CHECK-NEXT: [[TMP24:%.*]] = mul i64 [[TMP20]], [[TMP0]]
+; CHECK-NEXT: [[TMP36:%.*]] = mul <2 x i64> [[TMP16]], [[TMP85]]
; CHECK-NEXT: [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[DIFF_CHECK3783:%.*]] = icmp ult i64 [[TMP11]], [[TMP26]]
; CHECK-NEXT: [[DIFF_CHECK3790:%.*]] = icmp ult i64 [[INDVAR37888]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3791:%.*]] = or i1 [[DIFF_CHECK3783]], [[DIFF_CHECK3790]]
; CHECK-NEXT: [[TMP39:%.*]] = mul i64 [[TMP0]], [[TMP0]]
; CHECK-NEXT: [[DIFF_CHECK3793:%.*]] = icmp ult i64 [[TMP39]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3794:%.*]] = or i1 [[CONFLICT_RDX3791]], [[DIFF_CHECK3793]]
; CHECK-NEXT: [[TMP28:%.*]] = or i64 [[TMP0]], 1
; CHECK-NEXT: [[DIFF_CHECK3796:%.*]] = icmp ult i64 [[TMP28]], [[TMP1]]
-; CHECK-NEXT: [[CONFLICT_RDX3797:%.*]] = or i1 [[CONFLICT_RDX3794]], [[DIFF_CHECK3796]]
; CHECK-NEXT: [[TMP29:%.*]] = or i64 [[TMP0]], 1
; CHECK-NEXT: [[DIFF_CHECK3799:%.*]] = icmp ult i64 [[TMP29]], [[TMP1]]
-; CHECK-NEXT: [[CONFLICT_RDX3800:%.*]] = or i1 [[CONFLICT_RDX3797]], [[DIFF_CHECK3799]]
; CHECK-NEXT: [[TMP30:%.*]] = or i64 [[TMP0]], 1
; CHECK-NEXT: [[DIFF_CHECK3802:%.*]] = icmp ult i64 [[TMP30]], [[TMP1]]
-; CHECK-NEXT: [[CONFLICT_RDX3803:%.*]] = or i1 [[CONFLICT_RDX3800]], [[DIFF_CHECK3802]]
; CHECK-NEXT: [[DIFF_CHECK3805:%.*]] = icmp ugt i64 [[TMP26]], 1
-; CHECK-NEXT: [[CONFLICT_RDX3806:%.*]] = or i1 [[CONFLICT_RDX3803]], [[DIFF_CHECK3805]]
+; CHECK-NEXT: [[TMP22:%.*]] = extractelement <4 x i64> [[TMP34]], i32 0
; CHECK-NEXT: [[TMP31:%.*]] = add i64 [[TMP22]], [[TMP0]]
; CHECK-NEXT: [[DIFF_CHECK3807:%.*]] = icmp ult i64 [[TMP31]], [[DOTNEG1]]
-; CHECK-NEXT: [[CONFLICT_RDX3808:%.*]] = or i1 [[CONFLICT_RDX3806]], [[DIFF_CHECK3807]]
-; CHECK-NEXT: [[TMP32:%.*]] = add i64 [[TMP25]], 1
-; CHECK-NEXT: [[DIFF_CHECK3809:%.*]] = icmp ult i64 [[TMP32]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3810:%.*]] = or i1 [[CONFLICT_RDX3808]], [[DIFF_CHECK3809]]
-; CHECK-NEXT: [[TMP33:%.*]] = add i64 [[TMP25]], 1
-; CHECK-NEXT: [[DIFF_CHECK3811:%.*]] = icmp ult i64 [[TMP33]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3812:%.*]] = or i1 [[CONFLICT_RDX3810]], [[DIFF_CHECK3811]]
-; CHECK-NEXT: [[TMP34:%.*]] = add i64 [[TMP25]], 1
-; CHECK-NEXT: [[DIFF_CHECK3813:%.*]] = icmp ult i64 [[TMP34]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3814:%.*]] = or i1 [[CONFLICT_RDX3812]], [[DIFF_CHECK3813]]
-; CHECK-NEXT: [[TMP35:%.*]] = add i64 [[TMP25]], 1
-; CHECK-NEXT: [[DIFF_CHECK3815:%.*]] = icmp ult i64 [[TMP35]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3816:%.*]] = or i1 [[CONFLICT_RDX3814]], [[DIFF_CHECK3815]]
; CHECK-NEXT: [[DIFF_CHECK3817:%.*]] = icmp ult i64 [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[CONFLICT_RDX3818:%.*]] = or i1 [[CONFLICT_RDX3816]], [[DIFF_CHECK3817]]
-; CHECK-NEXT: [[TMP36:%.*]] = add i64 [[TMP23]], 1
-; CHECK-NEXT: [[DIFF_CHECK3820:%.*]] = icmp ult i64 [[TMP36]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3821:%.*]] = or i1 [[CONFLICT_RDX3818]], [[DIFF_CHECK3820]]
-; CHECK-NEXT: [[DIFF_CHECK3822:%.*]] = icmp ult i64 [[TMP25]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3823:%.*]] = or i1 [[CONFLICT_RDX3821]], [[DIFF_CHECK3822]]
-; CHECK-NEXT: [[TMP37:%.*]] = shl i64 [[TMP0]], 1
-; CHECK-NEXT: [[DIFF_CHECK3824:%.*]] = icmp ult i64 [[TMP37]], [[TMP1]]
-; CHECK-NEXT: [[CONFLICT_RDX3825:%.*]] = or i1 [[CONFLICT_RDX3823]], [[DIFF_CHECK3824]]
-; CHECK-NEXT: [[TMP38:%.*]] = shl i64 [[TMP0]], 1
-; CHECK-NEXT: [[DIFF_CHECK3826:%.*]] = icmp ult i64 [[TMP38]], [[TMP1]]
-; CHECK-NEXT: [[CONFLICT_RDX3827:%.*]] = or i1 [[CONFLICT_RDX3825]], [[DIFF_CHECK3826]]
-; CHECK-NEXT: [[DIFF_CHECK3830:%.*]] = icmp ult i64 [[TMP1]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3831:%.*]] = or i1 [[CONFLICT_RDX3827]], [[DIFF_CHECK3830]]
-; CHECK-NEXT: [[DIFF_CHECK3832:%.*]] = icmp ult i64 [[TMP10]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3833:%.*]] = or i1 [[CONFLICT_RDX3831]], [[DIFF_CHECK3832]]
-; CHECK-NEXT: [[DIFF_CHECK3834:%.*]] = icmp ult i64 [[INDVAR37888]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3835:%.*]] = or i1 [[CONFLICT_RDX3833]], [[DIFF_CHECK3834]]
-; CHECK-NEXT: [[TMP44:%.*]] = mul i64 [[TMP0]], [[TMP0]]
-; CHECK-NEXT: [[DIFF_CHECK3836:%.*]] = icmp ult i64 [[TMP44]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3837:%.*]] = or i1 [[CONFLICT_RDX3835]], [[DIFF_CHECK3836]]
-; CHECK-NEXT: [[TMP40:%.*]] = mul i64 [[TMP12]], [[TMP0]]
-; CHECK-NEXT: [[DIFF_CHECK3838:%.*]] = icmp ult i64 [[TMP40]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3839:%.*]] = or i1 [[CONFLICT_RDX3837]], [[DIFF_CHECK3838]]
-; CHECK-NEXT: [[TMP41:%.*]] = mul i64 [[TMP13]], [[TMP0]]
-; CHECK-NEXT: [[DIFF_CHECK3840:%.*]] = icmp ult i64 [[TMP41]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3841:%.*]] = or i1 [[CONFLICT_RDX3839]], [[DIFF_CHECK3840]]
; CHECK-NEXT: [[TMP42:%.*]] = add i64 [[TMP24]], 1
; CHECK-NEXT: [[DIFF_CHECK3842:%.*]] = icmp ult i64 [[TMP42]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3843:%.*]] = or i1 [[CONFLICT_RDX3841]], [[DIFF_CHECK3842]]
-; CHECK-NEXT: [[DIFF_CHECK3846:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[CONFLICT_RDX3847:%.*]] = or i1 [[CONFLICT_RDX3843]], [[DIFF_CHECK3846]]
-; CHECK-NEXT: [[DIFF_CHECK3848:%.*]] = icmp ult i64 [[TMP22]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3849:%.*]] = or i1 [[CONFLICT_RDX3847]], [[DIFF_CHECK3848]]
-; CHECK-NEXT: [[DIFF_CHECK3850:%.*]] = icmp ult i64 [[TMP25]], [[TMP1]]
-; CHECK-NEXT: [[CONFLICT_RDX3851:%.*]] = or i1 [[CONFLICT_RDX3849]], [[DIFF_CHECK3850]]
-; CHECK-NEXT: [[DIFF_CHECK3852:%.*]] = icmp ult i64 [[TMP1]], [[TMP4]]
-; CHECK-NEXT: [[CONFLICT_RDX3853:%.*]] = or i1 [[CONFLICT_RDX3851]], [[DIFF_CHECK3852]]
-; CHECK-NEXT: [[DIFF_CHECK3854:%.*]] = icmp ult i64 [[TMP1]], [[INDVAR3788]]
-; CHECK-NEXT: [[CONFLICT_RDX3855:%.*]] = or i1 [[CONFLICT_RDX3853]], [[DIFF_CHECK3854]]
+; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i64> [[TMP34]], i32 1
; CHECK-NEXT: [[DIFF_CHECK3856:%.*]] = icmp ult i64 [[TMP25]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3857:%.*]] = or i1 [[CONFLICT_RDX3855]], [[DIFF_CHECK3856]]
-; CHECK-NEXT: [[DIFF_CHECK3860:%.*]] = icmp ult i64 [[TMP1]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3861:%.*]] = or i1 [[CONFLICT_RDX3857]], [[DIFF_CHECK3860]]
-; CHECK-NEXT: [[DIFF_CHECK3862:%.*]] = icmp ult i64 [[TMP37]], [[TMP1]]
-; CHECK-NEXT: [[CONFLICT_RDX3863:%.*]] = or i1 [[CONFLICT_RDX3861]], [[DIFF_CHECK3862]]
+; CHECK-NEXT: [[TMP38:%.*]] = extractelement <4 x i64> [[TMP34]], i32 2
; CHECK-NEXT: [[DIFF_CHECK3864:%.*]] = icmp ult i64 [[TMP38]], [[TMP1]]
-; CHECK-NEXT: [[CONFLICT_RDX3865:%.*]] = or i1 [[CONFLICT_RDX3863]], [[DIFF_CHECK3864]]
-; CHECK-NEXT: [[DIFF_CHECK3866:%.*]] = icmp ult i64 [[TMP9]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3867:%.*]] = or i1 [[CONFLICT_RDX3865]], [[DIFF_CHECK3866]]
-; CHECK-NEXT: [[DIFF_CHECK3868:%.*]] = icmp ult i64 [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[CONFLICT_RDX3869:%.*]] = or i1 [[CONFLICT_RDX3867]], [[DIFF_CHECK3868]]
-; CHECK-NEXT: [[DIFF_CHECK3870:%.*]] = icmp ult i64 [[INDVAR37888]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3871:%.*]] = or i1 [[CONFLICT_RDX3869]], [[DIFF_CHECK3870]]
-; CHECK-NEXT: [[DIFF_CHECK3872:%.*]] = icmp ult i64 [[TMP44]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3873:%.*]] = or i1 [[CONFLICT_RDX3871]], [[DIFF_CHECK3872]]
-; CHECK-NEXT: [[DIFF_CHECK3876:%.*]] = icmp ult i64 [[TMP21]], [[TMP0]]
-; CHECK-NEXT: [[DIFF_CHECK3878:%.*]] = icmp ult i64 [[TMP24]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3877:%.*]] = or i1 [[CONFLICT_RDX3873]], [[DIFF_CHECK3876]]
-; CHECK-NEXT: [[CONFLICT_RDX3879:%.*]] = or i1 [[CONFLICT_RDX3877]], [[DIFF_CHECK3878]]
-; CHECK-NEXT: [[DIFF_CHECK3882:%.*]] = icmp ult i64 [[TMP1]], [[TMP5]]
-; CHECK-NEXT: [[CONFLICT_RDX3883:%.*]] = or i1 [[CONFLICT_RDX3879]], [[DIFF_CHECK3882]]
-; CHECK-NEXT: [[DIFF_CHECK3884:%.*]] = icmp ult i64 [[TMP1]], [[TMP6]]
-; CHECK-NEXT: [[CONFLICT_RDX3885:%.*]] = or i1 [[CONFLICT_RDX3883]], [[DIFF_CHECK3884]]
-; CHECK-NEXT: [[DIFF_CHECK3886:%.*]] = icmp ult i64 [[TMP22]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3887:%.*]] = or i1 [[CONFLICT_RDX3885]], [[DIFF_CHECK3886]]
-; CHECK-NEXT: [[DIFF_CHECK3890:%.*]] = icmp ult i64 [[TMP25]], [[TMP1]]
-; CHECK-NEXT: [[CONFLICT_RDX3891:%.*]] = or i1 [[CONFLICT_RDX3887]], [[DIFF_CHECK3890]]
-; CHECK-NEXT: [[DIFF_CHECK3892:%.*]] = icmp ult i64 [[TMP1]], [[TMP7]]
-; CHECK-NEXT: [[CONFLICT_RDX3893:%.*]] = or i1 [[CONFLICT_RDX3891]], [[DIFF_CHECK3892]]
-; CHECK-NEXT: [[DIFF_CHECK3894:%.*]] = icmp ult i64 [[TMP1]], [[DOTSROA_3341_0_COPYLOAD]]
-; CHECK-NEXT: [[CONFLICT_RDX3895:%.*]] = or i1 [[CONFLICT_RDX3893]], [[DIFF_CHECK3894]]
-; CHECK-NEXT: [[DIFF_CHECK3896:%.*]] = icmp ult i64 [[TMP25]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3897:%.*]] = or i1 [[CONFLICT_RDX3895]], [[DIFF_CHECK3896]]
+; CHECK-NEXT: [[TMP47:%.*]] = extractelement <4 x i64> [[TMP34]], i32 3
+; CHECK-NEXT: [[DIFF_CHECK3826:%.*]] = icmp ult i64 [[TMP47]], [[TMP1]]
; CHECK-NEXT: [[DIFF_CHECK3898:%.*]] = icmp ult i64 [[TMP1]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3899:%.*]] = or i1 [[CONFLICT_RDX3897]], [[DIFF_CHECK3898]]
-; CHECK-NEXT: [[DIFF_CHECK3900:%.*]] = icmp ult i64 [[TMP38]], [[TMP1]]
-; CHECK-NEXT: [[CONFLICT_RDX3901:%.*]] = or i1 [[CONFLICT_RDX3899]], [[DIFF_CHECK3900]]
-; CHECK-NEXT: [[DIFF_CHECK3902:%.*]] = icmp ult i64 [[TMP9]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3903:%.*]] = or i1 [[CONFLICT_RDX3901]], [[DIFF_CHECK3902]]
-; CHECK-NEXT: [[DIFF_CHECK3904:%.*]] = icmp ult i64 [[INDVAR37888]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3905:%.*]] = or i1 [[CONFLICT_RDX3903]], [[DIFF_CHECK3904]]
-; CHECK-NEXT: [[DIFF_CHECK3906:%.*]] = icmp ult i64 [[TMP44]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3907:%.*]] = or i1 [[CONFLICT_RDX3905]], [[DIFF_CHECK3906]]
-; CHECK-NEXT: [[TMP43:%.*]] = add i64 [[TMP24]], 1
+; CHECK-NEXT: [[DIFF_CHECK3832:%.*]] = icmp ult i64 [[TMP10]], [[TMP0]]
+; CHECK-NEXT: [[TMP48:%.*]] = mul <2 x i64> [[TMP12]], [[TMP13]]
+; CHECK-NEXT: [[TMP49:%.*]] = add <2 x i64> [[TMP36]], <i64 0, i64 1>
+; CHECK-NEXT: [[TMP50:%.*]] = insertelement <32 x i64> [[TMP74]], i64 [[INDVAR37888]], i32 0
+; CHECK-NEXT: [[TMP51:%.*]] = shufflevector <2 x i64> [[TMP48]], <2 x i64> poison, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP52:%.*]] = shufflevector <32 x i64> [[TMP50]], <32 x i64> [[TMP51]], <32 x i32> <i32 0, i32 32, i32 33, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT: [[TMP53:%.*]] = shufflevector <2 x i64> [[TMP49]], <2 x i64> poison, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP54:%.*]] = shufflevector <32 x i64> [[TMP52]], <32 x i64> [[TMP53]], <32 x i32> <i32 0, i32 1, i32 2, i32 32, i32 33, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT: [[TMP55:%.*]] = shufflevector <4 x i64> [[TMP34]], <4 x i64> poison, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP56:%.*]] = shufflevector <32 x i64> [[TMP54]], <32 x i64> [[TMP55]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 32, i32 33, i32 poison, i32 poison, i32 poison, i32 poison, i32 38, i32 39, i32 14, i32 poison, i32 poison, i32 poison, i32 18, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP57:%.*]] = shufflevector <2 x i64> [[TMP36]], <2 x i64> poison, <32 x i32> <i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP58:%.*]] = shufflevector <2 x i64> [[TMP36]], <2 x i64> poison, <32 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP59:%.*]] = shufflevector <32 x i64> [[TMP56]], <32 x i64> [[TMP58]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 12, i32 13, i32 14, i32 poison, i32 poison, i32 poison, i32 18, i32 33, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP60:%.*]] = shufflevector <32 x i64> [[TMP59]], <32 x i64> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 5, i32 5, i32 7, i32 5, i32 12, i32 13, i32 14, i32 5, i32 0, i32 1, i32 18, i32 19, i32 5, i32 5, i32 6, i32 7, i32 5, i32 5, i32 7, i32 5, i32 13, i32 14, i32 0, i32 1>
+; CHECK-NEXT: [[TMP61:%.*]] = shufflevector <32 x i64> [[TMP59]], <32 x i64> poison, <10 x i32> <i32 poison, i32 poison, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP62:%.*]] = insertelement <10 x i64> [[TMP61]], i64 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP63:%.*]] = insertelement <10 x i64> [[TMP62]], i64 [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP64:%.*]] = insertelement <10 x i64> [[TMP63]], i64 [[TMP4]], i32 3
+; CHECK-NEXT: [[TMP65:%.*]] = insertelement <10 x i64> [[TMP64]], i64 [[INDVAR3788]], i32 4
+; CHECK-NEXT: [[TMP66:%.*]] = insertelement <10 x i64> [[TMP65]], i64 [[TMP2]], i32 5
+; CHECK-NEXT: [[TMP67:%.*]] = insertelement <10 x i64> [[TMP66]], i64 [[TMP5]], i32 6
+; CHECK-NEXT: [[TMP68:%.*]] = insertelement <10 x i64> [[TMP67]], i64 [[TMP6]], i32 7
+; CHECK-NEXT: [[TMP69:%.*]] = insertelement <10 x i64> [[TMP68]], i64 [[TMP7]], i32 8
+; CHECK-NEXT: [[TMP70:%.*]] = insertelement <10 x i64> [[TMP69]], i64 [[DOTSROA_3341_0_COPYLOAD]], i32 9
+; CHECK-NEXT: [[TMP71:%.*]] = shufflevector <10 x i64> [[TMP70]], <10 x i64> poison, <32 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 0, i32 2, i32 3, i32 4, i32 0, i32 0, i32 2, i32 2, i32 0, i32 5, i32 0, i32 0, i32 0, i32 0, i32 6, i32 7, i32 0, i32 2, i32 8, i32 9, i32 0, i32 0, i32 2, i32 0, i32 0, i32 0>
+; CHECK-NEXT: [[TMP72:%.*]] = icmp ult <32 x i64> [[TMP60]], [[TMP71]]
+; CHECK-NEXT: [[TMP73:%.*]] = extractelement <2 x i64> [[TMP36]], i32 1
+; CHECK-NEXT: [[TMP43:%.*]] = add i64 [[TMP73]], 1
; CHECK-NEXT: [[DIFF_CHECK3912:%.*]] = icmp ult i64 [[TMP43]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3913:%.*]] = or i1 [[CONFLICT_RDX3907]], [[DIFF_CHECK3912]]
; CHECK-NEXT: [[DIFF_CHECK3916:%.*]] = icmp ult i64 [[TMP22]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3917:%.*]] = or i1 [[CONFLICT_RDX3913]], [[DIFF_CHECK3916]]
; CHECK-NEXT: [[DIFF_CHECK3920:%.*]] = icmp ult i64 [[TMP22]], [[TMP1]]
-; CHECK-NEXT: [[CONFLICT_RDX3921:%.*]] = or i1 [[CONFLICT_RDX3917]], [[DIFF_CHECK3920]]
; CHECK-NEXT: [[DIFF_CHECK3922:%.*]] = icmp ult i64 [[TMP22]], [[TMP2]]
-; CHECK-NEXT: [[CONFLICT_RDX3923:%.*]] = or i1 [[CONFLICT_RDX3921]], [[DIFF_CHECK3922]]
; CHECK-NEXT: [[DIFF_CHECK3924:%.*]] = icmp ult i64 [[TMP1]], [[TMP8]]
-; CHECK-NEXT: [[CONFLICT_RDX3925:%.*]] = or i1 [[CONFLICT_RDX3923]], [[DIFF_CHECK3924]]
; CHECK-NEXT: [[DIFF_CHECK3926:%.*]] = icmp ult i64 [[TMP1]], [[DOTSROA_3308_0_COPYLOAD]]
-; CHECK-NEXT: [[CONFLICT_RDX3927:%.*]] = or i1 [[CONFLICT_RDX3925]], [[DIFF_CHECK3926]]
-; CHECK-NEXT: [[DIFF_CHECK3928:%.*]] = icmp ult i64 [[TMP23]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3929:%.*]] = or i1 [[CONFLICT_RDX3927]], [[DIFF_CHECK3928]]
+; CHECK-NEXT: [[DIFF_CHECK3928:%.*]] = icmp ult i64 [[TMP24]], [[TMP0]]
; CHECK-NEXT: [[DIFF_CHECK3930:%.*]] = icmp ult i64 [[TMP25]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3931:%.*]] = or i1 [[CONFLICT_RDX3929]], [[DIFF_CHECK3930]]
-; CHECK-NEXT: [[DIFF_CHECK3932:%.*]] = icmp ult i64 [[TMP37]], [[TMP1]]
-; CHECK-NEXT: [[CONFLICT_RDX3933:%.*]] = or i1 [[CONFLICT_RDX3931]], [[DIFF_CHECK3932]]
+; CHECK-NEXT: [[DIFF_CHECK3932:%.*]] = icmp ult i64 [[TMP38]], [[TMP1]]
; CHECK-NEXT: [[DIFF_CHECK3934:%.*]] = icmp ult i64 [[TMP1]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3935:%.*]] = or i1 [[CONFLICT_RDX3933]], [[DIFF_CHECK3934]]
; CHECK-NEXT: [[DIFF_CHECK3936:%.*]] = icmp ult i64 [[INDVAR37888]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3937:%.*]] = or i1 [[CONFLICT_RDX3935]], [[DIFF_CHECK3936]]
-; CHECK-NEXT: [[DIFF_CHECK3938:%.*]] = icmp ult i64 [[TMP44]], [[TMP0]]
-; CHECK-NEXT: [[CONFLICT_RDX3939:%.*]] = or i1 [[CONFLICT_RDX3937]], [[DIFF_CHECK3938]]
+; CHECK-NEXT: [[DIFF_CHECK3938:%.*]] = icmp ult i64 [[TMP21]], [[TMP0]]
; CHECK-NEXT: [[DIFF_CHECK3950:%.*]] = icmp ult i64 [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP79:%.*]] = or i1 [[CONFLICT_RDX3939]], [[DIFF_CHECK3950]]
+; CHECK-NEXT: [[TMP75:%.*]] = call i1 @llvm.vector.reduce.or.v32i1(<32 x i1> [[TMP72]])
+; CHECK-NEXT: [[OP_RDX:%.*]] = or i1 [[TMP75]], [[DIFF_CHECK3783]]
+; CHECK-NEXT: [[OP_RDX1:%.*]] = or i1 [[DIFF_CHECK3790]], [[DIFF_CHECK3793]]
+; CHECK-NEXT: [[OP_RDX2:%.*]] = or i1 [[DIFF_CHECK3796]], [[DIFF_CHECK3799]]
+; CHECK-NEXT: [[OP_RDX3:%.*]] = or i1 [[DIFF_CHECK3802]], [[DIFF_CHECK3807]]
+; CHECK-NEXT: [[TMP76:%.*]] = shufflevector <4 x i64> [[TMP34]], <4 x i64> poison, <2 x i32> <i32 1, i32 poison>
+; CHECK-NEXT: [[TMP77:%.*]] = shufflevector <2 x i64> [[TMP76]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP78:%.*]] = add <2 x i64> [[TMP77]], splat (i64 1)
+; CHECK-NEXT: [[TMP86:%.*]] = icmp ult <2 x i64> [[TMP78]], [[TMP12]]
+; CHECK-NEXT: [[TMP80:%.*]] = add <2 x i64> [[TMP77]], splat (i64 1)
+; CHECK-NEXT: [[TMP81:%.*]] = icmp ult <2 x i64> [[TMP80]], [[TMP12]]
+; CHECK-NEXT: [[TMP82:%.*]] = or <2 x i1> [[TMP86]], [[TMP81]]
+; CHECK-NEXT: [[OP_RDX6:%.*]] = or i1 [[DIFF_CHECK3817]], [[DIFF_CHECK3842]]
+; CHECK-NEXT: [[OP_RDX7:%.*]] = or i1 [[DIFF_CHECK3856]], [[DIFF_CHECK3864]]
+; CHECK-NEXT: [[OP_RDX8:%.*]] = or i1 [[DIFF_CHECK3826]], [[DIFF_CHECK3898]]
+; CHECK-NEXT: [[OP_RDX9:%.*]] = or i1 [[DIFF_CHECK3832]], [[DIFF_CHECK3912]]
+; CHECK-NEXT: [[OP_RDX10:%.*]] = or i1 [[DIFF_CHECK3916]], [[DIFF_CHECK3920]]
+; CHECK-NEXT: [[OP_RDX11:%.*]] = or i1 [[DIFF_CHECK3922]], [[DIFF_CHECK3924]]
+; CHECK-NEXT: [[OP_RDX12:%.*]] = or i1 [[DIFF_CHECK3926]], [[DIFF_CHECK3928]]
+; CHECK-NEXT: [[OP_RDX13:%.*]] = or i1 [[DIFF_CHECK3930]], [[DIFF_CHECK3932]]
+; CHECK-NEXT: [[OP_RDX14:%.*]] = or i1 [[DIFF_CHECK3934]], [[DIFF_CHECK3936]]
+; CHECK-NEXT: [[OP_RDX15:%.*]] = or i1 [[DIFF_CHECK3938]], [[DIFF_CHECK3950]]
+; CHECK-NEXT: [[OP_RDX16:%.*]] = or i1 [[OP_RDX]], [[OP_RDX1]]
+; CHECK-NEXT: [[OP_RDX17:%.*]] = or i1 [[OP_RDX2]], [[OP_RDX3]]
+; CHECK-NEXT: [[TMP83:%.*]] = extractelement <2 x i1> [[TMP82]], i32 0
+; CHECK-NEXT: [[TMP84:%.*]] = extractelement <2 x i1> [[TMP82]], i32 1
+; CHECK-NEXT: [[OP_RDX18:%.*]] = or i1 [[TMP83]], [[TMP84]]
+; CHECK-NEXT: [[OP_RDX19:%.*]] = or i1 [[OP_RDX6]], [[OP_RDX7]]
+; CHECK-NEXT: [[OP_RDX20:%.*]] = or i1 [[OP_RDX8]], [[OP_RDX9]]
+; CHECK-NEXT: [[OP_RDX21:%.*]] = or i1 [[OP_RDX10]], [[OP_RDX11]]
+; CHECK-NEXT: [[OP_RDX22:%.*]] = or i1 [[OP_RDX12]], [[OP_RDX13]]
+; CHECK-NEXT: [[OP_RDX23:%.*]] = or i1 [[OP_RDX14]], [[OP_RDX15]]
+; CHECK-NEXT: [[OP_RDX24:%.*]] = or i1 [[OP_RDX16]], [[OP_RDX17]]
+; CHECK-NEXT: [[OP_RDX25:%.*]] = or i1 [[OP_RDX18]], [[OP_RDX19]]
+; CHECK-NEXT: [[OP_RDX26:%.*]] = or i1 [[OP_RDX20]], [[OP_RDX21]]
+; CHECK-NEXT: [[OP_RDX27:%.*]] = or i1 [[OP_RDX22]], [[OP_RDX23]]
+; CHECK-NEXT: [[OP_RDX28:%.*]] = or i1 [[OP_RDX24]], [[OP_RDX25]]
+; CHECK-NEXT: [[OP_RDX29:%.*]] = or i1 [[OP_RDX26]], [[OP_RDX27]]
+; CHECK-NEXT: [[OP_RDX30:%.*]] = or i1 [[OP_RDX28]], [[OP_RDX29]]
+; CHECK-NEXT: [[TMP79:%.*]] = or i1 [[OP_RDX30]], [[DIFF_CHECK3805]]
; CHECK-NEXT: br i1 [[TMP79]], label %[[SCALAR_PH4023:.*]], label %[[DOTLR_PH1977_US]]
; CHECK: [[SCALAR_PH4023]]:
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
index 6848ca42e064b..1cef13c9cf1b7 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
@@ -36,7 +36,7 @@
; YAML-NEXT: Function: getelementptr_4x32
; YAML-NEXT: Args:
; YAML-NEXT: - String: 'SLP vectorized with cost '
-; YAML-NEXT: - Cost: '12'
+; YAML-NEXT: - Cost: '10'
; YAML-NEXT: - String: ' and with tree size '
; YAML-NEXT: - TreeSize: '3'
@@ -47,6 +47,8 @@ define i32 @getelementptr_4x32(ptr nocapture readonly %g, i32 %n, i32 %x, i32 %y
; CHECK-NEXT: br i1 [[CMP31]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
; CHECK: for.body.preheader:
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[X:%.*]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> poison, i32 [[Y:%.*]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> [[TMP4]], i32 [[Z:%.*]], i32 1
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.cond.cleanup.loopexit:
; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]
@@ -66,10 +68,11 @@ define i32 @getelementptr_4x32(ptr nocapture readonly %g, i32 %n, i32 %x, i32 %y
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[TMP3]], i32 1
; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[G]], i32 [[TMP11]]
; CHECK-NEXT: [[T8:%.*]] = load i32, ptr [[ARRAYIDX5]], align 4
-; CHECK-NEXT: [[TMP13:%.*]] = add nsw i32 [[T4]], [[Y:%.*]]
+; CHECK-NEXT: [[TMP16:%.*]] = add nsw <2 x i32> [[TMP2]], [[TMP5]]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i32> [[TMP16]], i32 0
; CHECK-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, ptr [[G]], i32 [[TMP13]]
; CHECK-NEXT: [[T10:%.*]] = load i32, ptr [[ARRAYIDX10]], align 4
-; CHECK-NEXT: [[TMP14:%.*]] = add nsw i32 [[T4]], [[Z:%.*]]
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i32> [[TMP16]], i32 1
; CHECK-NEXT: [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, ptr [[G]], i32 [[TMP14]]
; CHECK-NEXT: [[T12:%.*]] = load i32, ptr [[ARRAYIDX15]], align 4
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[T6]], i32 0
@@ -128,7 +131,7 @@ for.body:
; YAML: Function: getelementptr_2x32
; YAML: Args:
; YAML: - String: 'SLP vectorized with cost '
-; YAML: - Cost: '12'
+; YAML: - Cost: '10'
; YAML-NEXT: - String: ' and with tree size '
; YAML-NEXT: - TreeSize: '3'
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/slp-fma-loss.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/slp-fma-loss.ll
index 8078fa924eebd..ba4c60c09ce38 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/slp-fma-loss.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/slp-fma-loss.ll
@@ -207,22 +207,21 @@ define float @slp_not_profitable_in_loop(float %x, ptr %A) {
; CHECK-LABEL: @slp_not_profitable_in_loop(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 1
-; CHECK-NEXT: [[L_0:%.*]] = load float, ptr [[GEP_A_1]], align 4
-; CHECK-NEXT: [[A1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
-; CHECK-NEXT: [[L_2:%.*]] = load float, ptr [[A1]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[GEP_A_1]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x float> [[TMP0]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
; CHECK-NEXT: [[L_3:%.*]] = load float, ptr [[A]], align 4
; CHECK-NEXT: [[L_4:%.*]] = load float, ptr [[A]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> <float 3.000000e+00, float 3.000000e+00, float poison, float 3.000000e+00>, float [[X:%.*]], i32 2
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x float> poison, float [[L_3]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x float> [[TMP3]], float [[L_4]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP5]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[RED:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[RED_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = fmul fast float 3.000000e+00, [[L_0]]
-; CHECK-NEXT: [[MUL12:%.*]] = fmul fast float 3.000000e+00, [[L_2]]
-; CHECK-NEXT: [[TMP4:%.*]] = fmul fast float [[X:%.*]], [[L_3]]
-; CHECK-NEXT: [[MUL16:%.*]] = fmul fast float 3.000000e+00, [[L_4]]
-; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[MUL12]], [[TMP3]]
-; CHECK-NEXT: [[ADD13:%.*]] = fadd fast float [[ADD]], [[TMP4]]
-; CHECK-NEXT: [[RED_NEXT]] = fadd fast float [[ADD13]], [[MUL16]]
+; CHECK-NEXT: [[TMP7:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP6]]
+; CHECK-NEXT: [[RED_NEXT]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP7]])
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV]], 10
; CHECK-NEXT: br i1 [[CMP]], label [[EXIT:%.*]], label [[LOOP]]
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/vec-elt-insertion.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/vec-elt-insertion.ll
index 199a53da4f70c..d50f2109298a5 100644
--- a/llvm/test/Transforms/SLPVectorizer/SystemZ/vec-elt-insertion.ll
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/vec-elt-insertion.ll
@@ -73,7 +73,7 @@ define void @fun1(double %0) {
; REMARK-LABEL: Function: fun1
; REMARK: Args:
; REMARK: - String: 'SLP vectorized with cost '
-; REMARK-NEXT: - Cost: '-2'
+; REMARK-NEXT: - Cost: '-3'
br label %2
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/buildvectors-parent-phi-nodes.ll b/llvm/test/Transforms/SLPVectorizer/X86/buildvectors-parent-phi-nodes.ll
index 16f31e3655de1..6e85c44711acd 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/buildvectors-parent-phi-nodes.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/buildvectors-parent-phi-nodes.ll
@@ -5,18 +5,18 @@ define void @test(ptr %0, float %1) {
; CHECK-LABEL: define void @test(
; CHECK-SAME: ptr [[TMP0:%.*]], float [[TMP1:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[TMP0]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x float> <float 0.000000e+00, float poison>, float [[TMP3]], i32 1
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x float> <float poison, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, float [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x float> poison, float [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[BB5:.*]]
; CHECK: [[BB5]]:
-; CHECK-NEXT: [[TMP6:%.*]] = phi float [ [[TMP1]], %[[BB5]] ], [ [[TMP3]], [[TMP2:%.*]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = phi float [ [[TMP1]], %[[BB5]] ], [ 0.000000e+00, [[TMP2]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x float> [ [[TMP15:%.*]], %[[BB5]] ], [ [[TMP5]], [[TMP2]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x float> [ [[TMP15:%.*]], %[[BB5]] ], [ [[TMP5]], [[TMP2:%.*]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = phi <2 x float> [ [[TMP7]], %[[BB5]] ], [ [[TMP4]], [[TMP2]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> [[TMP10]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 0>
; CHECK-NEXT: [[TMP12:%.*]] = fmul <4 x float> [[TMP9]], zeroinitializer
; CHECK-NEXT: [[TMP13:%.*]] = fadd <4 x float> [[TMP12]], zeroinitializer
; CHECK-NEXT: store <4 x float> [[TMP13]], ptr [[TMP0]], align 16
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x float> poison, float [[TMP7]], i32 0
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x float> [[TMP16]], float [[TMP6]], i32 1
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x float> [[TMP17]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 0>
; CHECK-NEXT: [[TMP14:%.*]] = fmul <4 x float> [[TMP11]], zeroinitializer
; CHECK-NEXT: [[TMP15]] = fadd <4 x float> [[TMP14]], zeroinitializer
; CHECK-NEXT: br label %[[BB5]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cse.ll b/llvm/test/Transforms/SLPVectorizer/X86/cse.ll
index 65241c13f3719..26ef502b81f05 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cse.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cse.ll
@@ -338,12 +338,9 @@ define void @cse_for_hoisted_instructions_in_preheader(ptr %dst, i32 %a, i1 %c)
; CHECK-NEXT: [[TMP2:%.*]] = or <2 x i32> splat (i32 22), [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = or <2 x i32> [[TMP2]], splat (i32 3)
; CHECK-NEXT: store <2 x i32> [[TMP3]], ptr [[DST:%.*]], align 4
-; CHECK-NEXT: [[OR_2:%.*]] = or i32 [[A]], 3
+; CHECK-NEXT: [[TMP4:%.*]] = or <2 x i32> [[TMP1]], splat (i32 3)
; CHECK-NEXT: [[GEP_2:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 10
-; CHECK-NEXT: store i32 [[OR_2]], ptr [[GEP_2]], align 4
-; CHECK-NEXT: [[OR_3:%.*]] = or i32 [[A]], 3
-; CHECK-NEXT: [[GEP_3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 11
-; CHECK-NEXT: store i32 [[OR_3]], ptr [[GEP_3]], align 4
+; CHECK-NEXT: store <2 x i32> [[TMP4]], ptr [[GEP_2]], align 4
; CHECK-NEXT: br i1 [[C:%.*]], label [[LOOP]], label [[EXIT:%.*]]
; CHECK: exit:
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll b/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll
index 5ad5366888bcb..1bb24c524bb3e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll
@@ -5,6 +5,8 @@ define void @test(i32 %arg, i32 %arg1, i64 %arg2) {
; CHECK-LABEL: define void @test(
; CHECK-SAME: i32 [[ARG:%.*]], i32 [[ARG1:%.*]], i64 [[ARG2:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[BB:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 poison, i32 poison>, i32 [[ARG]], i32 2
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[ARG1]], i32 3
; CHECK-NEXT: br label %[[BB3:.*]]
; CHECK: [[BB3]]:
; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ 0, %[[BB3]] ], [ 0, %[[BB]] ]
@@ -28,15 +30,14 @@ define void @test(i32 %arg, i32 %arg1, i64 %arg2) {
; CHECK-NEXT: [[TMP9:%.*]] = mul <4 x i32> [[TMP5]], [[TMP8]]
; CHECK-NEXT: [[XOR38:%.*]] = xor i32 [[ARG]], [[TRUNC28]]
; CHECK-NEXT: [[SHL35:%.*]] = shl i32 [[ARG1]], 0
-; CHECK-NEXT: [[XOR31:%.*]] = xor i32 [[ARG1]], [[TRUNC19]]
-; CHECK-NEXT: [[XOR37:%.*]] = xor i32 [[ARG]], [[TRUNC27]]
-; CHECK-NEXT: [[SHL17:%.*]] = shl i32 [[SHL]], 0
-; CHECK-NEXT: [[XOR:%.*]] = xor i32 0, [[TRUNC10]]
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> poison, i32 [[TRUNC10]], i32 0
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[SHL]], i32 1
+; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TRUNC27]], i32 2
+; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP24]], i32 [[TRUNC19]], i32 3
+; CHECK-NEXT: [[TMP26:%.*]] = xor <4 x i32> [[TMP11]], [[TMP25]]
; CHECK-NEXT: [[TMP23:%.*]] = insertelement <8 x i32> poison, i32 [[SHL]], i32 0
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <8 x i32> [[TMP23]], i32 [[XOR]], i32 1
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <8 x i32> [[TMP11]], i32 [[SHL17]], i32 2
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <8 x i32> [[TMP12]], i32 [[XOR37]], i32 3
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <8 x i32> [[TMP13]], i32 [[XOR31]], i32 5
+; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <4 x i32> [[TMP26]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 3, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <8 x i32> [[TMP23]], <8 x i32> [[TMP27]], <8 x i32> <i32 0, i32 8, i32 9, i32 10, i32 poison, i32 12, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <8 x i32> [[TMP14]], i32 [[SHL35]], i32 6
; CHECK-NEXT: [[TMP16:%.*]] = insertelement <8 x i32> [[TMP15]], i32 [[XOR38]], i32 7
; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <8 x i32> [[TMP16]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 5, i32 6, i32 7>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/deleted-node-with-copyable-operands.ll b/llvm/test/Transforms/SLPVectorizer/X86/deleted-node-with-copyable-operands.ll
index 310358b692724..c18f2a3e7da1a 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/deleted-node-with-copyable-operands.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/deleted-node-with-copyable-operands.ll
@@ -5,9 +5,12 @@ define void @test(ptr %0, ptr %1, i1 %cond, double %2) {
; CHECK-LABEL: define void @test(
; CHECK-SAME: ptr [[TMP0:%.*]], ptr [[TMP1:%.*]], i1 [[COND:%.*]], double [[TMP2:%.*]]) {
; CHECK-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> <double poison, double 0.000000e+00, double 0.000000e+00, double poison>, double [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 0>
+; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> <double 0.000000e+00, double 0.000000e+00, double poison, double 0.000000e+00>, <4 x i32> <i32 4, i32 5, i32 0, i32 7>
; CHECK-NEXT: br [[DOTLR_PH383_US_US_US_US:label %.*]]
; CHECK: [[_LR_PH383_US_US_US_US:.*:]]
-; CHECK-NEXT: [[TMP5:%.*]] = phi <4 x double> [ [[TMP18:%.*]], %[[DOT_CRIT_EDGE384_US_US_US_US:.*]] ], [ zeroinitializer, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x double> [ [[TMP22:%.*]], %[[DOT_CRIT_EDGE384_US_US_US_US:.*]] ], [ zeroinitializer, %[[ITER_CHECK]] ]
; CHECK-NEXT: br i1 false, label %[[DOTLR_PH383_US_US_US_US___CRIT_EDGE384_US_US_US_US_CRIT_EDGE:.*]], label %[[BB6:.*]]
; CHECK: [[_LR_PH383_US_US_US_US___CRIT_EDGE384_US_US_US_US_CRIT_EDGE:.*:]]
; CHECK-NEXT: br label %[[DOT_CRIT_EDGE384_US_US_US_US]]
@@ -17,23 +20,18 @@ define void @test(ptr %0, ptr %1, i1 %cond, double %2) {
; CHECK-NEXT: [[TMP9:%.*]] = load double, ptr [[TMP0]], align 8
; CHECK-NEXT: [[TMP10:%.*]] = load double, ptr [[TMP1]], align 8
; CHECK-NEXT: [[TMP11:%.*]] = fmul double [[TMP9]], 0.000000e+00
-; CHECK-NEXT: [[TMP12:%.*]] = fadd double [[TMP11]], 0.000000e+00
-; CHECK-NEXT: [[TMP16:%.*]] = fadd double [[TMP9]], 0.000000e+00
-; CHECK-NEXT: [[TMP17:%.*]] = fadd double [[TMP2]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x double> poison, double [[TMP9]], i32 0
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x double> [[TMP12]], double [[TMP10]], i32 2
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x double> [[TMP13]], double [[TMP11]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x double> [[TMP14]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 2, i32 3>
+; CHECK-NEXT: [[TMP21:%.*]] = fadd <4 x double> [[TMP20]], [[TMP5]]
; CHECK-NEXT: br label %[[BB16]]
; CHECK: [[BB16]]:
-; CHECK-NEXT: [[DOT0311_US_US_US_US:%.*]] = phi double [ [[TMP12]], %[[BB7]] ], [ [[TMP2]], %[[BB6]] ]
-; CHECK-NEXT: [[DOT0304_US_US_US_US:%.*]] = phi double [ [[TMP17]], %[[BB7]] ], [ 0.000000e+00, %[[BB6]] ]
-; CHECK-NEXT: [[DOT0301_US_US_US_US:%.*]] = phi double [ [[TMP16]], %[[BB7]] ], [ 0.000000e+00, %[[BB6]] ]
-; CHECK-NEXT: [[DOT0257_US_US_US_US:%.*]] = phi double [ [[TMP16]], %[[BB7]] ], [ [[TMP2]], %[[BB6]] ]
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x double> poison, double [[DOT0257_US_US_US_US]], i32 0
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x double> [[TMP13]], double [[DOT0301_US_US_US_US]], i32 1
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x double> [[TMP14]], double [[DOT0304_US_US_US_US]], i32 2
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x double> [[TMP15]], double [[DOT0311_US_US_US_US]], i32 3
-; CHECK-NEXT: [[TMP21:%.*]] = fadd <4 x double> [[TMP20]], [[TMP5]]
+; CHECK-NEXT: [[TMP18:%.*]] = phi <4 x double> [ [[TMP21]], %[[BB7]] ], [ [[TMP4]], %[[BB6]] ]
+; CHECK-NEXT: [[TMP19:%.*]] = fadd <4 x double> [[TMP18]], [[TMP6]]
; CHECK-NEXT: br label %[[DOT_CRIT_EDGE384_US_US_US_US]]
; CHECK: [[__CRIT_EDGE384_US_US_US_US:.*:]]
-; CHECK-NEXT: [[TMP18]] = phi <4 x double> [ [[TMP21]], %[[BB16]] ], [ zeroinitializer, %[[DOTLR_PH383_US_US_US_US___CRIT_EDGE384_US_US_US_US_CRIT_EDGE]] ]
+; CHECK-NEXT: [[TMP22]] = phi <4 x double> [ [[TMP19]], %[[BB16]] ], [ zeroinitializer, %[[DOTLR_PH383_US_US_US_US___CRIT_EDGE384_US_US_US_US_CRIT_EDGE]] ]
; CHECK-NEXT: br [[DOTLR_PH383_US_US_US_US]]
;
iter.check:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/parent-node-schedulable-with-multi-copyables.ll b/llvm/test/Transforms/SLPVectorizer/X86/parent-node-schedulable-with-multi-copyables.ll
index 8830561f9fec7..05611ba0151e6 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/parent-node-schedulable-with-multi-copyables.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/parent-node-schedulable-with-multi-copyables.ll
@@ -5,13 +5,13 @@ define i64 @test(ptr %arg1, i64 %alloca.promoted344, i8 %load.311.i, i1 %load1.i
; CHECK-LABEL: define i64 @test(
; CHECK-SAME: ptr [[ARG1:%.*]], i64 [[ALLOCA_PROMOTED344:%.*]], i8 [[LOAD_311_I:%.*]], i1 [[LOAD1_I:%.*]]) {
; CHECK-NEXT: [[BB:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <8 x i8> <i8 0, i8 0, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0>, i8 [[LOAD_311_I]], i32 2
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x i8> <i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison>, i8 [[LOAD_311_I]], i32 1
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i64> poison, i64 [[ALLOCA_PROMOTED344]], i32 1
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i8> <i8 0, i8 0, i8 0, i8 poison>, i8 [[LOAD_311_I]], i32 3
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i8> <i8 poison, i8 poison, i8 0, i8 0>, i8 [[LOAD_311_I]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i64> poison, i64 [[ALLOCA_PROMOTED344]], i32 0
; CHECK-NEXT: br label %[[BB2:.*]]
; CHECK: [[BB2]]:
; CHECK-NEXT: [[TMP3:%.*]] = phi <2 x i64> [ zeroinitializer, %[[BB]] ], [ [[TMP28:%.*]], %[[BB12_8_I:.*]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = phi <8 x i8> [ zeroinitializer, %[[BB]] ], [ [[TMP27:%.*]], %[[BB12_8_I]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi <8 x i8> [ zeroinitializer, %[[BB]] ], [ [[TMP29:%.*]], %[[BB12_8_I]] ]
; CHECK-NEXT: br i1 [[LOAD1_I]], label %[[SPAM_EXIT:.*]], label %[[BB4_LR_PH_I:.*]]
; CHECK: [[BB4_LR_PH_I]]:
; CHECK-NEXT: br i1 true, label %[[BB3_I_I_PEEL:.*]], label %[[EGGS_EXIT_I_PEEL:.*]]
@@ -19,26 +19,23 @@ define i64 @test(ptr %arg1, i64 %alloca.promoted344, i8 %load.311.i, i1 %load1.i
; CHECK-NEXT: [[TMP5:%.*]] = and <2 x i64> [[TMP3]], splat (i64 1)
; CHECK-NEXT: [[LOAD4_I_I_PEEL:%.*]] = load i64, ptr [[ARG1]], align 8
; CHECK-NEXT: [[SHL_I_I_PEEL:%.*]] = shl i64 [[LOAD4_I_I_PEEL]], 1
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> poison, <2 x i32> <i32 1, i32 poison>
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i64> [[TMP6]], i64 [[SHL_I_I_PEEL]], i32 1
-; CHECK-NEXT: [[TMP9:%.*]] = xor <2 x i64> [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> poison, <2 x i32> <i32 poison, i32 0>
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i64> [[TMP6]], i64 [[SHL_I_I_PEEL]], i32 0
; CHECK-NEXT: [[TMP22:%.*]] = or <2 x i64> [[TMP5]], [[TMP7]]
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x i64> [[TMP9]], <2 x i64> [[TMP22]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP11:%.*]] = trunc <2 x i64> [[TMP10]] to <2 x i8>
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <2 x i8> [[TMP11]], <2 x i8> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <4 x i8> [[TMP12]], <4 x i8> poison, <4 x i32> <i32 poison, i32 0, i32 1, i32 poison>
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <4 x i8> [[TMP13]], <4 x i8> poison, <4 x i32> <i32 poison, i32 1, i32 2, i32 2>
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x i64> [[TMP10]], i32 0
+; CHECK-NEXT: [[TMP9:%.*]] = xor <2 x i64> [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x i64> [[TMP22]], <2 x i64> [[TMP9]], <2 x i32> <i32 0, i32 3>
; CHECK-NEXT: br label %[[EGGS_EXIT_I_PEEL]]
; CHECK: [[EGGS_EXIT_I_PEEL]]:
-; CHECK-NEXT: [[LOAD5_I_I93_PEEL:%.*]] = phi i64 [ [[TMP15]], %[[BB3_I_I_PEEL]] ], [ 0, %[[BB4_LR_PH_I]] ]
-; CHECK-NEXT: [[TMP16:%.*]] = phi <4 x i8> [ [[TMP14]], %[[BB3_I_I_PEEL]] ], [ <i8 poison, i8 0, i8 0, i8 0>, %[[BB4_LR_PH_I]] ]
+; CHECK-NEXT: [[TMP11:%.*]] = phi <2 x i64> [ [[TMP10]], %[[BB3_I_I_PEEL]] ], [ zeroinitializer, %[[BB4_LR_PH_I]] ]
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <2 x i64> [[TMP11]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 0>
+; CHECK-NEXT: [[TMP13:%.*]] = trunc <4 x i64> [[TMP12]] to <4 x i8>
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i64> [[TMP12]], i32 1
; CHECK-NEXT: br label %[[SPAM_EXIT]]
; CHECK: [[SPAM_EXIT]]:
-; CHECK-NEXT: [[GETELEMENTPTR_I_I_PROMOTED346:%.*]] = phi i64 [ [[LOAD5_I_I93_PEEL]], %[[EGGS_EXIT_I_PEEL]] ], [ 0, %[[BB2]] ]
+; CHECK-NEXT: [[GETELEMENTPTR_I_I_PROMOTED346:%.*]] = phi i64 [ [[TMP14]], %[[EGGS_EXIT_I_PEEL]] ], [ 0, %[[BB2]] ]
; CHECK-NEXT: [[LOAD_8_I:%.*]] = phi i8 [ 0, %[[EGGS_EXIT_I_PEEL]] ], [ 1, %[[BB2]] ]
-; CHECK-NEXT: [[TMP23:%.*]] = phi <4 x i8> [ [[TMP16]], %[[EGGS_EXIT_I_PEEL]] ], [ <i8 poison, i8 0, i8 0, i8 0>, %[[BB2]] ]
-; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <4 x i8> [[TMP23]], <4 x i8> poison, <8 x i32> <i32 2, i32 3, i32 2, i32 1, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT: [[TMP15:%.*]] = phi <4 x i8> [ [[TMP13]], %[[EGGS_EXIT_I_PEEL]] ], [ zeroinitializer, %[[BB2]] ]
+; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <4 x i8> [[TMP15]], <4 x i8> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
; CHECK-NEXT: br i1 [[LOAD1_I]], label %[[BB12_8_I]], label %[[BB12_1_THREAD_I:.*]]
; CHECK: [[BB12_1_THREAD_I]]:
; CHECK-NEXT: [[TMP17:%.*]] = extractelement <8 x i8> [[TMP4]], i32 0
@@ -47,35 +44,38 @@ define i64 @test(ptr %arg1, i64 %alloca.promoted344, i8 %load.311.i, i1 %load1.i
; CHECK: [[BB8_3_I]]:
; CHECK-NEXT: br label %[[BB12_3_I]]
; CHECK: [[BB12_3_I]]:
-; CHECK-NEXT: [[TMP20:%.*]] = extractelement <8 x i8> [[TMP4]], i32 7
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <8 x i8> [[TMP4]], i32 1
; CHECK-NEXT: [[ICMP5_7_I:%.*]] = icmp eq i8 [[TMP20]], 0
; CHECK-NEXT: br i1 [[ICMP5_7_I]], label %[[BB12_4_I:.*]], label %[[BB8_4_I:.*]]
; CHECK: [[BB8_4_I]]:
; CHECK-NEXT: br label %[[BB12_4_I]]
; CHECK: [[BB12_4_I]]:
-; CHECK-NEXT: [[TMP19:%.*]] = extractelement <8 x i8> [[TMP4]], i32 5
+; CHECK-NEXT: [[TMP19:%.*]] = extractelement <8 x i8> [[TMP4]], i32 2
; CHECK-NEXT: [[ICMP5_5_I:%.*]] = icmp eq i8 [[TMP19]], 0
; CHECK-NEXT: br i1 [[ICMP5_5_I]], label %[[BB12_5_I:.*]], label %[[BB8_5_I:.*]]
; CHECK: [[BB8_5_I]]:
; CHECK-NEXT: br label %[[BB12_5_I]]
; CHECK: [[BB12_5_I]]:
-; CHECK-NEXT: [[TMP21:%.*]] = extractelement <8 x i8> [[TMP4]], i32 1
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <8 x i8> [[TMP4]], i32 3
; CHECK-NEXT: [[ICMP5_8_I:%.*]] = icmp eq i8 [[TMP21]], 0
; CHECK-NEXT: br i1 [[ICMP5_8_I]], label %[[BB12_7_I:.*]], label %[[BB8_7_I:.*]]
; CHECK: [[BB8_7_I]]:
; CHECK-NEXT: br label %[[BB12_7_I]]
; CHECK: [[BB12_7_I]]:
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <8 x i8> [[TMP4]], i32 3
+; CHECK-NEXT: [[TMP30:%.*]] = extractelement <8 x i8> [[TMP4]], i32 4
; CHECK-NEXT: [[ICMP5_8_I1:%.*]] = icmp eq i8 [[TMP30]], 0
; CHECK-NEXT: br i1 [[ICMP5_8_I1]], label %[[BB12_8_I]], label %[[BB8_8_I:.*]]
; CHECK: [[BB8_8_I]]:
-; CHECK-NEXT: [[TMP24:%.*]] = insertelement <8 x i8> [[TMP1]], i8 [[LOAD_8_I]], i32 3
-; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <8 x i8> [[TMP24]], <8 x i8> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 10, i32 12, i32 14>
-; CHECK-NEXT: [[TMP26:%.*]] = shufflevector <8 x i8> [[TMP25]], <8 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 3, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i8> [[TMP1]], i8 [[LOAD_8_I]], i32 1
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i8> poison, i8 [[LOAD_8_I]], i32 0
+; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <8 x i8> [[TMP4]], <8 x i8> poison, <4 x i32> <i32 poison, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <4 x i8> [[TMP23]], <4 x i8> [[TMP24]], <4 x i32> <i32 0, i32 5, i32 6, i32 7>
; CHECK-NEXT: br label %[[BB12_8_I]]
; CHECK: [[BB12_8_I]]:
-; CHECK-NEXT: [[TMP27]] = phi <8 x i8> [ [[TMP0]], %[[BB12_7_I]] ], [ [[TMP26]], %[[BB8_8_I]] ], [ [[TMP18]], %[[SPAM_EXIT]] ]
-; CHECK-NEXT: [[TMP28]] = insertelement <2 x i64> [[TMP2]], i64 [[GETELEMENTPTR_I_I_PROMOTED346]], i32 0
+; CHECK-NEXT: [[TMP26:%.*]] = phi <4 x i8> [ [[TMP0]], %[[BB12_7_I]] ], [ [[TMP31]], %[[BB8_8_I]] ], [ [[TMP15]], %[[SPAM_EXIT]] ]
+; CHECK-NEXT: [[TMP27:%.*]] = phi <4 x i8> [ zeroinitializer, %[[BB12_7_I]] ], [ [[TMP25]], %[[BB8_8_I]] ], [ [[TMP16]], %[[SPAM_EXIT]] ]
+; CHECK-NEXT: [[TMP28]] = insertelement <2 x i64> [[TMP2]], i64 [[GETELEMENTPTR_I_I_PROMOTED346]], i32 1
+; CHECK-NEXT: [[TMP29]] = shufflevector <4 x i8> [[TMP26]], <4 x i8> [[TMP27]], <8 x i32> <i32 2, i32 7, i32 5, i32 0, i32 1, i32 3, i32 4, i32 6>
; CHECK-NEXT: br label %[[BB2]]
;
bb:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/parent-node-split-non-schedulable.ll b/llvm/test/Transforms/SLPVectorizer/X86/parent-node-split-non-schedulable.ll
index a6db17412be6f..bb912bc7c9713 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/parent-node-split-non-schedulable.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/parent-node-split-non-schedulable.ll
@@ -5,63 +5,60 @@ define i32 @main(ptr %c, i32 %0, i1 %tobool4.not, i16 %1) {
; CHECK-LABEL: define i32 @main(
; CHECK-SAME: ptr [[C:%.*]], i32 [[TMP0:%.*]], i1 [[TOBOOL4_NOT:%.*]], i16 [[TMP1:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> <i32 poison, i32 poison, i32 1, i32 0>, i32 [[TMP0]], i32 0
; CHECK-NEXT: br label %[[IF_END:.*]]
; CHECK: [[IF_END]]:
; CHECK-NEXT: [[B_0_PH:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[TMP32:%.*]], %[[WHILE_COND_PREHEADER:.*]] ]
; CHECK-NEXT: [[TMP3:%.*]] = phi <2 x i32> [ zeroinitializer, %[[ENTRY]] ], [ [[TMP33:%.*]], %[[WHILE_COND_PREHEADER]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> <i32 poison, i32 1>, i32 [[B_0_PH]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[B_0_PH]], i32 1
; CHECK-NEXT: br i1 [[TOBOOL4_NOT]], label %[[R:.*]], label %[[IF_END9:.*]]
; CHECK: [[IF_END9]]:
; CHECK-NEXT: [[CONV11:%.*]] = sext i16 [[TMP1]] to i32
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 [[CONV11]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> <i32 0, i32 poison, i32 0, i32 1>, i32 [[CONV11]], i32 1
; CHECK-NEXT: br label %[[R]]
; CHECK: [[R]]:
-; CHECK-NEXT: [[TMP13:%.*]] = phi <2 x i32> [ <i32 1, i32 0>, %[[IF_END9]] ], [ [[TMP2]], %[[IF_END]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i32> [ [[TMP5]], %[[IF_END9]] ], [ [[TMP4]], %[[IF_END]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i32> [ [[TMP5]], %[[IF_END9]] ], [ [[TMP4]], %[[IF_END]] ]
; CHECK-NEXT: [[TOBOOL12_NOT:%.*]] = icmp eq i32 [[B_0_PH]], 0
; CHECK-NEXT: br i1 [[TOBOOL12_NOT]], label %[[IF_END14:.*]], label %[[IF_THEN13:.*]]
; CHECK: [[IF_THEN13]]:
; CHECK-NEXT: br label %[[IF_END14]]
; CHECK: [[IF_END14]]:
+; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[C]], align 4
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i32> [[TMP3]], i32 1
; CHECK-NEXT: [[AND:%.*]] = and i32 [[TMP8]], 1
+; CHECK-NEXT: [[AND25:%.*]] = and i32 [[TMP0]], 1
; CHECK-NEXT: [[NOT:%.*]] = xor i32 [[AND]], 1
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[TMP3]], i32 0
; CHECK-NEXT: [[AND17:%.*]] = and i32 [[TMP9]], 1
; CHECK-NEXT: [[DIV20:%.*]] = sdiv i32 [[AND17]], [[TMP0]]
-; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[C]], align 4
-; CHECK-NEXT: [[AND25:%.*]] = and i32 [[TMP0]], 1
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[AND17]], i32 1
; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP10]], i32 2
; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[AND25]], i32 3
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> <i32 0, i32 1, i32 poison, i32 1>, i32 [[DIV20]], i32 2
; CHECK-NEXT: [[TMP16:%.*]] = xor <4 x i32> [[TMP14]], [[TMP15]]
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> poison, i32 [[NOT]], i32 2
-; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <4 x i32> [[TMP17]], <4 x i32> [[TMP6]], <4 x i32> <i32 poison, i32 poison, i32 2, i32 5>
-; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <2 x i32> [[TMP13]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <4 x i32> [[TMP18]], <4 x i32> [[TMP19]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> <i32 poison, i32 0, i32 0, i32 0>, i32 [[B_0_PH]], i32 0
-; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 poison, i32 0>, i32 [[TMP0]], i32 2
+; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <4 x i32> [[TMP16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <8 x i32> [[TMP22]], <8 x i32> [[TMP23]], <8 x i32> <i32 0, i32 1, i32 2, i32 8, i32 9, i32 10, i32 11, i32 poison>
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <8 x i32> [[TMP17]], i32 [[NOT]], i32 7
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <8 x i32> <i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 poison>, i32 [[B_0_PH]], i32 0
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <8 x i32> [[TMP19]], i32 [[TMP0]], i32 7
; CHECK-NEXT: br label %[[AH:.*]]
; CHECK: [[AH]]:
-; CHECK-NEXT: [[TMP23:%.*]] = phi <4 x i32> [ [[TMP21]], %[[AH]] ], [ [[TMP16]], %[[IF_END14]] ]
-; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x i32> [ [[TMP22]], %[[AH]] ], [ [[TMP20]], %[[IF_END14]] ]
-; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i32> [[TMP23]], i32 2
-; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i32> [[TMP24]], i32 2
+; CHECK-NEXT: [[TMP21:%.*]] = phi <8 x i32> [ [[TMP20]], %[[AH]] ], [ [[TMP18]], %[[IF_END14]] ]
+; CHECK-NEXT: [[TMP25:%.*]] = extractelement <8 x i32> [[TMP21]], i32 5
+; CHECK-NEXT: [[TMP26:%.*]] = extractelement <8 x i32> [[TMP21]], i32 7
; CHECK-NEXT: [[ADD:%.*]] = add i32 [[TMP25]], [[TMP26]]
-; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i32> [[TMP23]], i32 1
+; CHECK-NEXT: [[TMP28:%.*]] = extractelement <8 x i32> [[TMP21]], i32 4
; CHECK-NEXT: [[TMP29:%.*]] = or i32 [[ADD]], [[TMP28]]
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i32> [[TMP23]], i32 3
+; CHECK-NEXT: [[TMP30:%.*]] = extractelement <8 x i32> [[TMP21]], i32 6
; CHECK-NEXT: [[OR27:%.*]] = or i32 [[TMP29]], [[TMP30]]
; CHECK-NEXT: store i32 [[OR27]], ptr [[C]], align 4
; CHECK-NEXT: br i1 [[TOBOOL4_NOT]], label %[[WHILE_COND_PREHEADER]], label %[[AH]]
; CHECK: [[WHILE_COND_PREHEADER]]:
-; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i32> [[TMP24]], i32 3
+; CHECK-NEXT: [[TMP31:%.*]] = extractelement <8 x i32> [[TMP21]], i32 1
; CHECK-NEXT: [[CALL69:%.*]] = tail call i32 @s(i32 [[TMP31]])
-; CHECK-NEXT: [[TMP32]] = extractelement <4 x i32> [[TMP23]], i32 0
-; CHECK-NEXT: [[TMP33]] = shufflevector <4 x i32> [[TMP24]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP32]] = extractelement <8 x i32> [[TMP21]], i32 0
+; CHECK-NEXT: [[TMP33]] = shufflevector <8 x i32> [[TMP21]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
; CHECK-NEXT: br label %[[IF_END]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduced-val-extracted-and-externally-used.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduced-val-extracted-and-externally-used.ll
index 93db0d1c6087d..f740308ddee24 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduced-val-extracted-and-externally-used.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduced-val-extracted-and-externally-used.ll
@@ -5,37 +5,28 @@ define void @test(i32 %arg) {
; CHECK-LABEL: define void @test(
; CHECK-SAME: i32 [[ARG:%.*]]) {
; CHECK-NEXT: [[BB:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 [[ARG]], i32 0
; CHECK-NEXT: br label %[[BB1:.*]]
; CHECK: [[BB1]]:
; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[TMP5:%.*]], %[[BB1]] ]
-; CHECK-NEXT: [[PHI2:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[ADD26:%.*]], %[[BB1]] ]
+; CHECK-NEXT: [[PHI2:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[TMP6:%.*]], %[[BB1]] ]
; CHECK-NEXT: [[PHI3:%.*]] = phi i32 [ 0, %[[BB]] ], [ [[XOR27:%.*]], %[[BB1]] ]
-; CHECK-NEXT: [[ADD13:%.*]] = add i32 [[PHI2]], 0
-; CHECK-NEXT: [[ADD18:%.*]] = add i32 [[PHI2]], 0
-; CHECK-NEXT: [[XOR:%.*]] = xor i32 [[ADD13]], [[ADD18]]
-; CHECK-NEXT: [[ADD5:%.*]] = add i32 [[PHI]], 0
-; CHECK-NEXT: [[ADD19:%.*]] = add i32 [[PHI2]], 0
-; CHECK-NEXT: [[ADD7:%.*]] = add i32 [[PHI2]], 0
-; CHECK-NEXT: [[XOR8:%.*]] = xor i32 [[ADD19]], [[XOR]]
-; CHECK-NEXT: [[XOR9:%.*]] = xor i32 [[XOR8]], [[ADD5]]
-; CHECK-NEXT: [[XOR22:%.*]] = xor i32 [[XOR9]], [[ADD7]]
-; CHECK-NEXT: [[ADD24:%.*]] = add i32 [[PHI]], 0
-; CHECK-NEXT: [[ADD25:%.*]] = add i32 [[PHI2]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x i32> [ zeroinitializer, %[[BB]] ], [ [[TMP4:%.*]], %[[BB1]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <8 x i32> <i32 0, i32 0, i32 1, i32 0, i32 0, i32 1, i32 0, i32 0>
; CHECK-NEXT: [[ADD14:%.*]] = add i32 [[PHI2]], 0
-; CHECK-NEXT: [[XOR25:%.*]] = xor i32 [[ADD25]], [[XOR22]]
-; CHECK-NEXT: [[XOR26:%.*]] = xor i32 [[XOR25]], [[ADD24]]
-; CHECK-NEXT: [[XOR16:%.*]] = xor i32 [[XOR26]], [[ADD14]]
-; CHECK-NEXT: [[ADD17:%.*]] = add i32 [[PHI]], 0
; CHECK-NEXT: [[ADD20:%.*]] = add i32 [[PHI2]], 0
+; CHECK-NEXT: [[ADD5:%.*]] = add i32 [[PHI]], 0
; CHECK-NEXT: [[ADD21:%.*]] = add i32 [[PHI2]], 0
-; CHECK-NEXT: [[XOR20:%.*]] = xor i32 [[ADD20]], [[XOR16]]
-; CHECK-NEXT: [[XOR21:%.*]] = xor i32 [[XOR20]], [[ADD17]]
-; CHECK-NEXT: [[XOR23:%.*]] = xor i32 [[XOR21]], [[ADD21]]
-; CHECK-NEXT: [[ADD23:%.*]] = add i32 [[PHI2]], 0
-; CHECK-NEXT: [[ADD26]] = add i32 [[ARG]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = add <8 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT: [[TMP4]] = add <2 x i32> [[TMP0]], <i32 0, i32 1>
+; CHECK-NEXT: [[TMP5]] = extractelement <2 x i32> [[TMP4]], i32 1
+; CHECK-NEXT: [[TMP6]] = extractelement <2 x i32> [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> [[TMP3]])
+; CHECK-NEXT: [[ADD23:%.*]] = xor i32 [[TMP7]], [[ADD14]]
+; CHECK-NEXT: [[XOR23:%.*]] = xor i32 [[ADD20]], [[ADD21]]
+; CHECK-NEXT: [[ADD26:%.*]] = xor i32 [[ADD5]], [[TMP6]]
; CHECK-NEXT: [[XOR28:%.*]] = xor i32 [[ADD23]], [[XOR23]]
; CHECK-NEXT: [[XOR27]] = xor i32 [[XOR28]], [[ADD26]]
-; CHECK-NEXT: [[TMP5]] = add i32 1, 0
; CHECK-NEXT: [[ICMP:%.*]] = icmp ult i32 [[TMP5]], 0
; CHECK-NEXT: br label %[[BB1]]
;
>From 9a608ba8239332ffb2b1e1086c6b31300dfcf49d Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sat, 18 Apr 2026 12:51:07 -0700
Subject: [PATCH 2/2] Address comments
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 19 ++++++++++++++++---
1 file changed, 16 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 41b1464f07f94..173f4a9e4418e 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -15995,19 +15995,32 @@ uint64_t BoUpSLP::getGatherNodeEffectiveScale(const TreeEntry &TE) {
// lanes), which matches the LICM hoisting performed by
// optimizeGatherSequence(). Cap per-lane contributions by BaseScale so a
// refinement can never raise the cost above the whole-entry scale.
+ // Each lane contributes at most BaseScale, so Sum is bounded above by
+ // N * BaseScale. If BaseScale is near uint64_t max (saturated by
+ // getLoopNestScale on a deep nest) Sum can still overflow uint64_t,
+ // which would silently wrap and produce a wrong average. Use
+ // SaturatingAdd and bail out to BaseScale on overflow: the true average
+ // is bounded above by BaseScale anyway, so this preserves the
+ // refinement's invariant that it can never raise cost.
uint64_t Sum = 0;
unsigned N = 0;
+ bool Overflow = false;
for (Value *V : TE.Scalars) {
if (isConstant(V))
continue;
++N;
- uint64_t LaneScale = getScaleToLoopIterations(TE, V);
- Sum += std::min(LaneScale, BaseScale);
+ uint64_t LaneScale = std::min(getScaleToLoopIterations(TE, V), BaseScale);
+ Sum = SaturatingAdd(Sum, LaneScale, &Overflow);
+ if (Overflow)
+ return BaseScale;
}
if (N == 0)
return BaseScale;
// Ceil-divide so we never round the effective scale down below 1.
- uint64_t Avg = (Sum + N - 1) / N;
+ uint64_t Numerator = SaturatingAdd(Sum, uint64_t(N - 1), &Overflow);
+ if (Overflow)
+ return BaseScale;
+ uint64_t Avg = Numerator / N;
return std::clamp<uint64_t>(Avg, 1, BaseScale);
}
More information about the llvm-commits
mailing list