[llvm] [VPlan] Model initial header mask as region value. (PR #196199)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 29 04:34:45 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/196199
>From f96de23233d0d398d987a88eb9b63e8053a6dd95 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 24 May 2026 14:35:54 +0100
Subject: [PATCH 1/3] [VPlan] Make VPRegionValue a subclass of VPSymbolicValue
(NFC).
---
llvm/lib/Transforms/Vectorize/VPlan.cpp | 8 +-
.../Transforms/Vectorize/VPlanPatternMatch.h | 12 ++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 6 +-
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 18 ++---
llvm/lib/Transforms/Vectorize/VPlanValue.h | 74 +++++++++----------
.../Transforms/Vectorize/VPlanTest.cpp | 26 +++++++
7 files changed, 93 insertions(+), 53 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 9c0bec2350c97..1e55f2834ef8c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1234,17 +1234,19 @@ VPlan *VPlan::duplicate() {
// else NewTripCount will be created and inserted into Old2NewVPValues when
// TripCount is cloned. In any case NewPlan->TripCount is updated below.
+ assert(none_of(Old2NewVPValues.keys(), IsaPred<VPSymbolicValue>) &&
+ "All VPSymbolicValues must be handled below");
+
if (auto *LoopRegion = getVectorLoopRegion()) {
auto *OldCanIV = LoopRegion->getCanonicalIV();
auto *NewCanIV = NewPlan->getVectorLoopRegion()->getCanonicalIV();
assert(OldCanIV && NewCanIV &&
"Loop regions of both plans must have canonical IVs.");
Old2NewVPValues[OldCanIV] = NewCanIV;
+ if (OldCanIV->isMaterialized())
+ NewCanIV->markMaterialized();
}
- assert(none_of(Old2NewVPValues.keys(), IsaPred<VPSymbolicValue>) &&
- "All VPSymbolicValues must be handled below");
-
if (BackedgeTakenCount)
NewPlan->BackedgeTakenCount =
new VPSymbolicValue(BackedgeTakenCount->getType());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index e2e08c50d06e8..124534c5e75a0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -1077,7 +1077,17 @@ inline auto m_WidenIntrinsic(const T &...Ops) {
return m_Isa<VPWidenIntrinsicRecipe>(m_Intrinsic<IntrID>(Ops...));
}
-inline auto m_LiveIn() { return m_Isa<VPIRValue, VPSymbolicValue>(); }
+/// Match VPValues that represent live-ins: VPIRValues and (plain)
+/// VPSymbolicValues. VPRegionValues (which inherit from VPSymbolicValue) are
+/// not live-ins and are excluded.
+struct LiveIn_match {
+ template <typename ITy> bool match(ITy *V) const {
+ return isa<VPIRValue>(V) ||
+ (isa<VPSymbolicValue>(V) && !isa<VPRegionValue>(V));
+ }
+};
+
+inline LiveIn_match m_LiveIn() { return {}; }
/// Match a GEP recipe (VPWidenGEPRecipe, VPInstruction, or VPReplicateRecipe)
/// and bind the source element type and operands.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 46c63bb47b165..ea50994e38f3a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3450,8 +3450,10 @@ void VPlanTransforms::addExplicitVectorLength(
// Replace all uses of the canonical IV with VPCurrentIterationPHIRecipe
// except for the canonical IV increment.
- CanonicalIV->replaceAllUsesWith(CurrentIteration);
- CanonicalIVIncrement->setOperand(0, CanonicalIV);
+ CanonicalIV->replaceUsesWithIf(CurrentIteration,
+ [CanonicalIVIncrement](VPUser &U, unsigned) {
+ return &U != CanonicalIVIncrement;
+ });
// TODO: support unroll factor > 1.
Plan.setUF(1);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index bcd17a54a3e31..594c201d5a8cb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -78,7 +78,7 @@ class UnrollState {
void unrollBlock(VPBlockBase *VPB);
VPValue *getValueForPart(VPValue *V, unsigned Part) {
- if (Part == 0 || isa<VPIRValue, VPSymbolicValue, VPRegionValue>(V))
+ if (Part == 0 || isa<VPIRValue, VPSymbolicValue>(V))
return V;
assert((VPV2Parts.contains(V) && VPV2Parts[V].size() >= Part) &&
"accessed value does not exist");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index c6b83e69ee3d8..812a63fd830a7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -171,13 +171,6 @@ const SCEV *vputils::getSCEVExprForVPValue(const VPValue *V,
PredicatedScalarEvolution &PSE,
const Loop *L) {
ScalarEvolution &SE = *PSE.getSE();
- if (isa<VPIRValue, VPSymbolicValue>(V)) {
- Value *LiveIn = V->getUnderlyingValue();
- if (LiveIn && SE.isSCEVable(LiveIn->getType()))
- return SE.getSCEV(LiveIn);
- return SE.getCouldNotCompute();
- }
-
if (auto *RV = dyn_cast<VPRegionValue>(V)) {
assert(RV == RV->getDefiningRegion()->getCanonicalIV() &&
"RegionValue must be canonical IV");
@@ -187,6 +180,13 @@ const SCEV *vputils::getSCEVExprForVPValue(const VPValue *V,
L, SCEV::FlagAnyWrap);
}
+ if (isa<VPIRValue, VPSymbolicValue>(V)) {
+ Value *LiveIn = V->getUnderlyingValue();
+ if (LiveIn && SE.isSCEVable(LiveIn->getType()))
+ return SE.getSCEV(LiveIn);
+ return SE.getCouldNotCompute();
+ }
+
// Helper to create SCEVs for binary and unary operations.
auto CreateSCEV = [&](ArrayRef<VPValue *> Ops,
function_ref<const SCEV *(ArrayRef<SCEVUse>)> CreateFn)
@@ -421,7 +421,7 @@ static bool preservesUniformity(unsigned Opcode) {
bool vputils::isSingleScalar(const VPValue *VPV) {
// Live-in, symbolic and region-values represent single-scalar values.
- if (isa<VPIRValue, VPSymbolicValue, VPRegionValue>(VPV))
+ if (isa<VPIRValue, VPSymbolicValue>(VPV))
return true;
if (auto *Rep = dyn_cast<VPReplicateRecipe>(VPV)) {
@@ -458,7 +458,7 @@ bool vputils::isSingleScalar(const VPValue *VPV) {
bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
// Live-ins and region values are uniform.
- if (isa<VPIRValue, VPSymbolicValue, VPRegionValue>(V))
+ if (isa<VPIRValue, VPSymbolicValue>(V))
return true;
const VPRecipeBase *R = V->getDefiningRecipe();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index a7aa0523ad5d0..49b612ea41fca 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -49,7 +49,7 @@ class VPSingleDefRecipe;
/// coming from the input IR, symbolic values and values defined by recipes.
class LLVM_ABI_FOR_TEST VPValue {
friend struct VPIRValue;
- friend struct VPSymbolicValue;
+ friend class VPSymbolicValue;
friend class VPRecipeValue;
friend class VPRegionValue;
@@ -212,15 +212,48 @@ class LLVM_ABI_FOR_TEST VPValue {
}
};
+/// A symbolic live-in VPValue, used for values like vector trip count, VF, and
+/// VFxUF.
+class VPSymbolicValue : public VPValue {
+ /// The scalar type of this symbolic value.
+ Type *Ty;
+
+ /// Track whether this value has been materialized (replaced). After
+ /// materialization, accessing users should trigger an assertion.
+ bool Materialized = false;
+
+protected:
+ VPSymbolicValue(unsigned char SC, Type *Ty) : VPValue(SC, nullptr), Ty(Ty) {}
+
+public:
+ VPSymbolicValue(Type *Ty) : VPSymbolicValue(VPVSymbolicSC, Ty) {}
+
+ /// Returns the scalar type of this symbolic value.
+ Type *getType() const { return Ty; }
+
+ /// Returns true if this value has been materialized.
+ bool isMaterialized() const { return Materialized; }
+
+ /// Mark this value as materialized.
+ void markMaterialized() {
+ assert(!Materialized && "VPSymbolicValue already materialized");
+ Materialized = true;
+ }
+
+ static bool classof(const VPValue *V) {
+ return V->getVPValueID() == VPVSymbolicSC ||
+ V->getVPValueID() == VPRegionValueSC;
+ }
+};
+
/// VPValues defined by a VPRegionBlock, like the canonical IV.
-class VPRegionValue : public VPValue {
+class VPRegionValue : public VPSymbolicValue {
VPRegionBlock *DefiningRegion;
- Type *Ty;
DebugLoc DL;
public:
VPRegionValue(Type *Ty, DebugLoc DL, VPRegionBlock *Region)
- : VPValue(VPValue::VPRegionValueSC), DefiningRegion(Region), Ty(Ty),
+ : VPSymbolicValue(VPValue::VPRegionValueSC, Ty), DefiningRegion(Region),
DL(DL) {}
~VPRegionValue() override = default;
@@ -228,9 +261,6 @@ class VPRegionValue : public VPValue {
/// Returns the region that defines this value.
VPRegionBlock *getDefiningRegion() const { return DefiningRegion; }
- /// Returns the type of the VPRegionValue.
- Type *getType() const { return Ty; }
-
/// Returns the debug location of the VPRegionValue.
DebugLoc getDebugLoc() const { return DL; }
@@ -282,36 +312,6 @@ struct VPConstantInt : public VPIRValue {
uint64_t getZExtValue() const { return getAPInt().getZExtValue(); }
};
-/// A symbolic live-in VPValue, used for values like vector trip count, VF, and
-/// VFxUF.
-struct VPSymbolicValue : public VPValue {
- VPSymbolicValue(Type *Ty) : VPValue(VPVSymbolicSC, nullptr), Ty(Ty) {}
-
- static bool classof(const VPValue *V) {
- return V->getVPValueID() == VPVSymbolicSC;
- }
-
- /// Returns the scalar type of this symbolic value.
- Type *getType() const { return Ty; }
-
- /// Returns true if this symbolic value has been materialized.
- bool isMaterialized() const { return Materialized; }
-
- /// Mark this symbolic value as materialized.
- void markMaterialized() {
- assert(!Materialized && "VPSymbolicValue already materialized");
- Materialized = true;
- }
-
-private:
- /// The scalar type of this symbolic value.
- Type *Ty;
-
- /// Track whether this symbolic value has been materialized (replaced).
- /// After materialization, accessing users should trigger an assertion.
- bool Materialized = false;
-};
-
/// Abstract base class for VPValues defined by a VPRecipeBase.
class VPRecipeValue : public VPValue {
friend class VPValue;
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 2deb7c6c864b0..db85bd2f2ef10 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1894,6 +1894,32 @@ TEST_F(VPUtilsTest, IsUniformAcrossVFsAndUFsForSingleScalarOpcodes) {
vputils::isUniformAcrossVFsAndUFs(FirstActiveLaneNonUniform.get()));
}
+TEST_F(VPBasicBlockTest, VPRegionValueClonePropagatesMaterialized) {
+ VPlan &Plan = getPlan();
+ VPBasicBlock *Preheader = Plan.getEntry();
+ VPBasicBlock *Header = Plan.createVPBasicBlock("header");
+ VPBasicBlock *Latch = Plan.createVPBasicBlock("latch");
+ VPRegionBlock *Region = Plan.createLoopRegion(Type::getInt32Ty(C), DebugLoc(),
+ "loop", Header, Latch);
+ VPBlockUtils::connectBlocks(Header, Latch);
+ VPBlockUtils::connectBlocks(Preheader, Region);
+ VPBlockUtils::connectBlocks(Region, Plan.getScalarHeader());
+
+ VPRegionValue *CanIV = Region->getCanonicalIV();
+ EXPECT_TRUE(isa<VPSymbolicValue>(CanIV));
+ EXPECT_FALSE(CanIV->isMaterialized());
+
+ // Materialize the canonical IV by replacing all uses, then verify clone
+ // propagates the materialized state.
+ CanIV->replaceAllUsesWith(Plan.getConstantInt(32, 0));
+ EXPECT_TRUE(CanIV->isMaterialized());
+
+ std::unique_ptr<VPlan> Clone(Plan.duplicate());
+ VPRegionValue *ClonedCanIV = Clone->getVectorLoopRegion()->getCanonicalIV();
+ EXPECT_NE(CanIV, ClonedCanIV);
+ EXPECT_TRUE(ClonedCanIV->isMaterialized());
+}
+
#if defined(GTEST_HAS_DEATH_TEST) && !defined(NDEBUG)
TEST_F(VPInstructionTest, VPSymbolicValueConstructUserAfterMaterialization) {
VPlan &Plan = getPlan();
>From c5f537531f24d5643d30f5f73dd1d8a571ed04c6 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 25 May 2026 15:21:45 +0100
Subject: [PATCH 2/3] [VPlan] Model initial abstract header mask as region
value.
Introduce a new VPRegionValue for the header mask, managed by loop
regions similar to the canonical IV.
The main benefit is we do not need to materialize the mask early and it is
trivial to find (no more need for patterns hard-coded in
findHeaderMask).
It is currently materialized before computing costs, as we compute costs
for part of it currently.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 14 +-
llvm/lib/Transforms/Vectorize/VPlan.cpp | 26 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 48 ++-
.../Vectorize/VPlanConstruction.cpp | 11 +-
.../Transforms/Vectorize/VPlanPatternMatch.h | 10 +
.../Transforms/Vectorize/VPlanTransforms.cpp | 89 +++---
.../Transforms/Vectorize/VPlanTransforms.h | 11 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 99 +------
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 11 -
.../Transforms/Vectorize/VPlanVerifier.cpp | 14 +-
.../AArch64/conditional-branches-cost.ll | 2 +-
.../AArch64/fold-tail-low-trip-count.ll | 4 +-
.../AArch64/force-target-instruction-cost.ll | 2 +-
.../AArch64/induction-costs-sve.ll | 2 +-
.../LoopVectorize/AArch64/predicated-costs.ll | 80 ++---
.../AArch64/reduction-recurrence-costs-sve.ll | 2 +-
.../LoopVectorize/AArch64/store-costs-sve.ll | 2 +-
.../widen-gep-all-indices-invariant.ll | 2 +-
.../LoopVectorize/ARM/mve-saddsatcost.ll | 2 +-
.../ARM/tail-folding-counting-down.ll | 16 +-
.../SystemZ/force-target-instruction-cost.ll | 2 +-
.../predicated-first-order-recurrence.ll | 2 +-
.../VPlan/X86/vplan-vp-intrinsics.ll | 2 +-
.../conditional-scalar-assignment-vplan.ll | 54 ++--
...-order-recurrence-sink-replicate-region.ll | 98 +++----
.../LoopVectorize/VPlan/tail-folding.ll | 102 +++----
.../VPlan/vplan-print-before-after-all.ll | 1 +
.../VPlan/vplan-printing-alias-mask.ll | 22 +-
.../vplan-printing-reductions-tail-folded.ll | 274 +++++++++---------
.../VPlan/vplan-sink-scalars-and-merge.ll | 166 +++++------
.../LoopVectorize/X86/cost-model.ll | 2 +-
...bounds-flags-for-reverse-vector-pointer.ll | 2 +-
.../X86/fold-tail-low-trip-count.ll | 4 +-
.../LoopVectorize/X86/induction-costs.ll | 2 +-
.../Transforms/LoopVectorize/X86/pr81872.ll | 2 +-
.../LoopVectorize/X86/small-size.ll | 8 +-
.../X86/vectorize-interleaved-accesses-gap.ll | 2 +-
.../Transforms/LoopVectorize/alias-mask.ll | 2 +-
...find-last-iv-sinkable-expr-tail-folding.ll | 10 +-
.../first-order-recurrence-tail-folding.ll | 6 +-
.../LoopVectorize/first-order-recurrence.ll | 24 +-
.../LoopVectorize/iv-select-cmp-fold-tail.ll | 2 +-
llvm/test/Transforms/LoopVectorize/optsize.ll | 10 +-
.../pr51614-fold-tail-by-masking.ll | 2 +-
.../LoopVectorize/reduction-order.ll | 2 +-
.../LoopVectorize/select-reduction.ll | 8 +-
...e-reduction-results-in-tail-folded-loop.ll | 2 +-
.../LoopVectorize/tail-folding-div.ll | 2 +-
.../tail-folding-iv-outside-user.ll | 2 +-
.../tail-folding-replicate-region.ll | 2 +-
.../tail-folding-vectorization-factor-1.ll | 22 +-
.../use-scalar-epilogue-if-tp-fails.ll | 4 +-
...-loop-backedge-elimination-tail-folding.ll | 2 +-
.../PhaseOrdering/ARM/arm_add_q7.ll | 2 +-
54 files changed, 620 insertions(+), 676 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 60fe8fda120b4..44991bfe6f9a0 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6631,6 +6631,11 @@ void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
RUN_VPLAN_PASS(VPlanTransforms::narrowInterleaveGroups, *Plan, TTI))
VPlans.push_back(std::move(P));
+ TailFoldingStyle Style = CM.getTailFoldingStyle();
+ RUN_VPLAN_PASS(VPlanTransforms::materializeHeaderMask, *Plan,
+ useActiveLaneMask(Style),
+ useActiveLaneMaskForControlFlow(Style));
+
RUN_VPLAN_PASS_NO_VERIFY(printOptimizedVPlan, *Plan);
assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
VPlans.push_back(std::move(Plan));
@@ -6871,13 +6876,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
RUN_VPLAN_PASS(VPlanTransforms::dropPoisonGeneratingRecipes, *Plan);
- if (useActiveLaneMask(Style)) {
- // TODO: Move checks to VPlanTransforms::addActiveLaneMask once
- // TailFoldingStyle is visible there.
- bool ForControlFlow = useActiveLaneMaskForControlFlow(Style);
- RUN_VPLAN_PASS(VPlanTransforms::addActiveLaneMask, *Plan, ForControlFlow);
- }
-
if (CM.maskPartialAliasing())
RUN_VPLAN_PASS(VPlanTransforms::attachAliasMaskToHeaderMask, *Plan);
@@ -6893,7 +6891,7 @@ void LoopVectorizationPlanner::addReductionResultComputation(
VPBasicBlock *LatchVPBB = VectorLoopRegion->getExitingBasicBlock();
Builder.setInsertPoint(&*std::prev(std::prev(LatchVPBB->end())));
VPBasicBlock::iterator IP = MiddleVPBB->getFirstNonPhi();
- VPValue *HeaderMask = vputils::findHeaderMask(*Plan);
+ VPValue *HeaderMask = Plan->getVectorLoopRegion()->getHeaderMask();
for (VPRecipeBase &R :
Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
VPReductionPHIRecipe *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 1e55f2834ef8c..74f1f419e7a97 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -744,6 +744,9 @@ VPRegionBlock *VPRegionBlock::clone() {
getName(), NewEntry, NewExiting)
: Plan.createReplicateRegion(NewEntry, NewExiting, getName());
+ if (getHeaderMask())
+ NewRegion->createHeaderMask();
+
for (VPBlockBase *Block : vp_depth_first_shallow(NewEntry))
Block->setParent(NewRegion);
return NewRegion;
@@ -818,6 +821,10 @@ void VPRegionBlock::print(raw_ostream &O, const Twine &Indent,
CanIV->print(O, SlotTracker);
O << " = CANONICAL-IV\n";
}
+ if (auto *HdrMask = getUsedHeaderMask()) {
+ HdrMask->print(O, SlotTracker);
+ O << " = HEADER-MASK\n";
+ }
for (auto *BlockBase : vp_depth_first_shallow(Entry)) {
O << '\n';
BlockBase->print(O, NewIndent, SlotTracker);
@@ -1238,13 +1245,13 @@ VPlan *VPlan::duplicate() {
"All VPSymbolicValues must be handled below");
if (auto *LoopRegion = getVectorLoopRegion()) {
- auto *OldCanIV = LoopRegion->getCanonicalIV();
- auto *NewCanIV = NewPlan->getVectorLoopRegion()->getCanonicalIV();
- assert(OldCanIV && NewCanIV &&
- "Loop regions of both plans must have canonical IVs.");
- Old2NewVPValues[OldCanIV] = NewCanIV;
- if (OldCanIV->isMaterialized())
- NewCanIV->markMaterialized();
+ auto *NewLoopRegion = NewPlan->getVectorLoopRegion();
+ for (auto [Old, New] : zip_equal(LoopRegion->getRegionValues(),
+ NewLoopRegion->getRegionValues())) {
+ Old2NewVPValues[Old] = New;
+ if (Old->isMaterialized())
+ New->markMaterialized();
+ }
}
if (BackedgeTakenCount)
@@ -1567,8 +1574,9 @@ void VPSlotTracker::assignNames(const VPlan &Plan) {
for (const VPBlockBase *VPB : RPOT) {
if (auto *VPBB = dyn_cast<VPBasicBlock>(VPB))
assignNames(VPBB);
- else if (auto *CanIV = cast<VPRegionBlock>(VPB)->getCanonicalIV())
- assignName(CanIV);
+ else
+ for (auto *RV : cast<VPRegionBlock>(VPB)->getRegionValues())
+ assignName(RV);
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index c043be05c6bd1..4a71228c5a9ba 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -4571,13 +4571,16 @@ class VPIRBasicBlock : public VPBasicBlock {
BasicBlock *getIRBasicBlock() const { return IRBB; }
};
-/// Track information about the canonical IV value of a region.
+/// Track information about the canonical IV and header mask of a loop region.
/// TODO: Have it also track the canonical IV increment, subject of NUW flag.
class VPCanonicalIVInfo {
/// VPRegionValue for the canonical IV, whose allocation is managed by
/// VPCanonicalIVInfo.
std::unique_ptr<VPRegionValue> CanIV;
+ /// Optional VPRegionValue for the header mask, set when tail folding.
+ std::unique_ptr<VPRegionValue> HeaderMask;
+
/// Whether the increment of the canonical IV may unsigned wrap or not.
bool HasNUW = true;
@@ -4588,6 +4591,18 @@ class VPCanonicalIVInfo {
VPRegionValue *getRegionValue() { return CanIV.get(); }
const VPRegionValue *getRegionValue() const { return CanIV.get(); }
+ VPRegionValue *getHeaderMask() const { return HeaderMask.get(); }
+
+ /// Create the header mask for the region and return it. Must only be called
+ /// when no header mask exists yet.
+ VPRegionValue *createHeaderMask() {
+ assert(!HeaderMask && "Header mask already created");
+ HeaderMask = std::make_unique<VPRegionValue>(
+ Type::getInt1Ty(CanIV->getType()->getContext()), DebugLoc::getUnknown(),
+ CanIV->getDefiningRegion());
+ return HeaderMask.get();
+ }
+
bool hasNUW() const { return HasNUW; }
void clearNUW() { HasNUW = false; }
@@ -4725,6 +4740,37 @@ class LLVM_ABI_FOR_TEST VPRegionBlock : public VPBlockBase {
return CanIVInfo->getRegionValue()->getType();
}
+ /// Return the header mask of the region, or null if not set.
+ VPRegionValue *getHeaderMask() const {
+ return CanIVInfo ? CanIVInfo->getHeaderMask() : nullptr;
+ }
+
+ /// Return the header mask if it exists and is used, or null otherwise. The
+ /// mask is materialized into concrete recipes only after costing, so cost and
+ /// codegen accounting sites use this to skip an unused mask.
+ VPRegionValue *getUsedHeaderMask() const {
+ VPRegionValue *HeaderMask = getHeaderMask();
+ return HeaderMask && HeaderMask->getNumUsers() > 0 ? HeaderMask : nullptr;
+ }
+
+ /// Create the header mask for the region and return it. Must only be called
+ /// on loop regions that don't already have a header mask.
+ VPRegionValue *createHeaderMask() {
+ assert(CanIVInfo && "Can only create header mask for loop regions");
+ return CanIVInfo->createHeaderMask();
+ }
+
+ /// Return the region values of the loop region (canonical IV, header mask)
+ /// or an empty vector for replicate regions.
+ SmallVector<VPRegionValue *, 2> getRegionValues() const {
+ if (!CanIVInfo)
+ return {};
+ SmallVector<VPRegionValue *, 2> R = {CanIVInfo->getRegionValue()};
+ if (auto *HM = CanIVInfo->getHeaderMask())
+ R.push_back(HM);
+ return R;
+ }
+
/// Indicates if NUW is set for the canonical IV increment, for loop regions.
bool hasCanonicalIVNUW() const { return CanIVInfo->hasNUW(); }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 619fea8c10b4d..cd09f0e2cebc9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1366,14 +1366,9 @@ void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
Header->splitAt(Header->getFirstNonPhi());
- // Create the header mask, insert it in the header and branch on it.
- auto *IV = new VPWidenCanonicalIVRecipe(
- LoopRegion->getCanonicalIV(),
- VPIRFlags::WrapFlagsTy(LoopRegion->hasCanonicalIVNUW(), false));
+ // Abstract header mask, materialized into concrete recipes later.
+ VPValue *HeaderMask = LoopRegion->createHeaderMask();
VPBuilder Builder(Header, Header->getFirstNonPhi());
- Builder.insert(IV);
- VPValue *BTC = Plan.getOrCreateBackedgeTakenCount();
- VPValue *HeaderMask = Builder.createICmp(CmpInst::ICMP_ULE, IV, BTC);
Builder.createNaryOp(VPInstruction::BranchOnCond, HeaderMask);
VPBasicBlock *OrigLatch = LoopRegion->getExitingBasicBlock();
@@ -1833,7 +1828,7 @@ bool VPlanTransforms::handleFindLastReductions(VPlan &Plan) {
if (Phis.empty())
return true;
- VPValue *HeaderMask = vputils::findHeaderMask(Plan);
+ VPValue *HeaderMask = Plan.getVectorLoopRegion()->getHeaderMask();
for (VPReductionPHIRecipe *PhiR : Phis) {
// Find the condition for the select/blend.
VPValue *BackedgeSelect = PhiR->getBackedgeValue();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 124534c5e75a0..6f61ddd75638c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -876,6 +876,16 @@ struct canonical_iv_match {
inline canonical_iv_match m_CanonicalIV() { return {}; }
+/// Match the abstract header mask of any loop region.
+struct header_mask_match {
+ template <typename ArgTy> bool match(const ArgTy *V) const {
+ const auto *RV = dyn_cast<VPRegionValue>(V);
+ return RV && RV->getDefiningRegion()->getHeaderMask() == RV;
+ }
+};
+
+inline header_mask_match m_HeaderMask() { return {}; }
+
/// Match a canonical VPWidenIntOrFpInductionRecipe optionally capturing it.
struct canonical_widen_iv_match {
VPWidenIntOrFpInductionRecipe **Capture = nullptr;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ea50994e38f3a..9896f2c745135 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1484,11 +1484,17 @@ static void simplifyRecipe(VPSingleDefRecipe *Def) {
// Drop the mask of a predicated store masked by the header mask (which is
// guaranteed to be true at least for the first lane) and both the stored
- // value and the address are uniform across VF and UF.
+ // value and the address are uniform across VF and UF. The header mask is
+ // still the abstract region value here, possibly clamped by an alias mask.
if (auto *RepR = dyn_cast<VPReplicateRecipe>(Def);
RepR && RepR->isPredicated() && RepR->getOpcode() == Instruction::Store &&
all_of(RepR->operandsWithoutMask(), vputils::isUniformAcrossVFsAndUFs) &&
- vputils::isHeaderMask(RepR->getMask(), *Plan)) {
+ match(RepR->getMask(),
+ m_CombineOr(
+ m_HeaderMask(),
+ m_c_BinaryAnd(
+ m_HeaderMask(),
+ m_VPInstruction<VPInstruction::IncomingAliasMask>())))) {
auto *Unmasked = new VPReplicateRecipe(
RepR->getUnderlyingInstr(), RepR->operandsWithoutMask(),
RepR->isSingleScalar(), /*Mask=*/nullptr, *RepR, *RepR,
@@ -1870,7 +1876,7 @@ void VPlanTransforms::simplifyReverses(VPlan &Plan) {
/// header mask to be simplified further when tail folding, e.g. in
/// optimizeEVLMasks.
static void reassociateHeaderMask(VPlan &Plan) {
- VPValue *HeaderMask = vputils::findHeaderMask(Plan);
+ VPValue *HeaderMask = Plan.getVectorLoopRegion()->getHeaderMask();
if (!HeaderMask)
return;
@@ -2948,32 +2954,36 @@ addVPLaneMaskPhiAndUpdateExitBranch(VPlan &Plan) {
return LaneMaskPhi;
}
-void VPlanTransforms::addActiveLaneMask(VPlan &Plan,
- bool UseActiveLaneMaskForControlFlow) {
+void VPlanTransforms::materializeHeaderMask(
+ VPlan &Plan, bool UseActiveLaneMask, bool UseActiveLaneMaskForControlFlow) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
- auto *WideCanonicalIV =
- findUserOf<VPWidenCanonicalIVRecipe>(LoopRegion->getCanonicalIV());
- assert(WideCanonicalIV &&
- "Must have widened canonical IV when tail folding!");
- VPSingleDefRecipe *HeaderMask = vputils::findHeaderMask(Plan);
- VPSingleDefRecipe *LaneMask;
+ VPValue *HeaderMask = LoopRegion->getUsedHeaderMask();
+ if (!HeaderMask)
+ return;
+
if (UseActiveLaneMaskForControlFlow) {
- LaneMask = addVPLaneMaskPhiAndUpdateExitBranch(Plan);
- } else {
- VPBuilder B = VPBuilder::getToInsertAfter(WideCanonicalIV);
+ HeaderMask->replaceAllUsesWith(addVPLaneMaskPhiAndUpdateExitBranch(Plan));
+ return;
+ }
+
+ VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
+ VPBuilder Builder(Header, Header->getFirstNonPhi());
+ auto *WideCanonicalIV = Builder.insert(new VPWidenCanonicalIVRecipe(
+ LoopRegion->getCanonicalIV(),
+ VPIRFlags::WrapFlagsTy(/*HasNUW=*/true, /*HasNSW=*/false)));
+ VPValue *Mask;
+ if (UseActiveLaneMask) {
VPValue *ALMMultiplier =
Plan.getConstantInt(LoopRegion->getCanonicalIVType(), 1);
- LaneMask =
- B.createNaryOp(VPInstruction::ActiveLaneMask,
- {WideCanonicalIV, Plan.getTripCount(), ALMMultiplier},
- nullptr, "active.lane.mask");
+ Mask = Builder.createNaryOp(
+ VPInstruction::ActiveLaneMask,
+ {WideCanonicalIV, Plan.getTripCount(), ALMMultiplier}, nullptr,
+ "active.lane.mask");
+ } else {
+ Mask = Builder.createICmp(CmpInst::ICMP_ULE, WideCanonicalIV,
+ Plan.getOrCreateBackedgeTakenCount());
}
-
- // Walk users of WideCanonicalIV and replace the header mask of the form
- // (ICMP_ULE, WideCanonicalIV, backedge-taken-count) with an active-lane-mask,
- // removing the old one to ensure there is always only a single header mask.
- HeaderMask->replaceAllUsesWith(LaneMask);
- HeaderMask->eraseFromParent();
+ HeaderMask->replaceAllUsesWith(Mask);
}
template <typename Op0_t, typename Op1_t> struct RemoveMask_match {
@@ -3313,7 +3323,7 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
}
}
- VPValue *HeaderMask = vputils::findHeaderMask(Plan);
+ VPValue *HeaderMask = LoopRegion->getHeaderMask();
if (!HeaderMask)
return;
@@ -3328,11 +3338,8 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
return true;
}));
- // Replace header masks with a mask equivalent to predicating by EVL:
- //
- // icmp ule widen-canonical-iv backedge-taken-count
- // ->
- // icmp ult step-vector, EVL
+ // Replace the abstract header mask with a mask equivalent to predicating by
+ // EVL: icmp ult step-vector, EVL
VPRecipeBase *EVLR = EVL.getDefiningRecipe();
VPBuilder Builder(EVLR->getParent(), std::next(EVLR->getIterator()));
Type *EVLType = EVL.getScalarType();
@@ -3674,9 +3681,13 @@ void VPlanTransforms::dropPoisonGeneratingRecipes(VPlan &Plan) {
// We want to exclude the tail folding case, as we don't need to drop flags
// for operations computing the first lane in this case: the first lane of the
- // header mask must always be true.
- auto IsNotHeaderMask = [&Plan](VPValue *Mask) {
- return Mask && !vputils::isHeaderMask(Mask, Plan);
+ // header mask must always be true. For reverse memory accesses, the mask is
+ // wrapped in a Reverse, which is just a permutation of the header mask, so
+ // peel it off before checking. The header mask is still the abstract region
+ // value at this point (materialization happens later).
+ auto IsNotHeaderMask = [](VPValue *Mask) {
+ return Mask &&
+ !match(Mask, m_CombineOr(m_HeaderMask(), m_Reverse(m_HeaderMask())));
};
// Traverse all the recipes in the VPlan and collect the poison-generating
@@ -5562,8 +5573,8 @@ void VPlanTransforms::materializeFactors(VPlan &Plan, VPBasicBlock *VectorPH,
}
void VPlanTransforms::attachAliasMaskToHeaderMask(VPlan &Plan) {
- VPSingleDefRecipe *HeaderMask = vputils::findHeaderMask(Plan);
- auto *HeaderMaskDef = HeaderMask->getDefiningRecipe();
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ VPValue *HeaderMask = LoopRegion->getHeaderMask();
Type *I1Ty = IntegerType::getInt1Ty(Plan.getContext());
VPBuilder Builder(Plan.getVectorPreheader());
@@ -5571,10 +5582,8 @@ void VPlanTransforms::attachAliasMaskToHeaderMask(VPlan &Plan) {
VPInstruction::IncomingAliasMask, {}, nullptr, {}, {},
DebugLoc::getUnknown(), "incoming.alias.mask", I1Ty);
- if (HeaderMaskDef->isPhi())
- Builder = VPBuilder(&*HeaderMaskDef->getParent()->getFirstNonPhi());
- else
- Builder = VPBuilder::getToInsertAfter(HeaderMaskDef);
+ VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
+ Builder = VPBuilder(Header, Header->getFirstNonPhi());
// Update all existing users of the header mask to "HeaderMask & AliasMask".
auto *ClampedHeaderMask = Builder.createAnd(HeaderMask, AliasMask);
@@ -6323,7 +6332,7 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
return std::nullopt;
};
- VPValue *HeaderMask = vputils::findHeaderMask(Plan);
+ VPValue *HeaderMask = VectorLoopRegion->getHeaderMask();
for (VPRecipeBase &Phi :
make_early_inc_range(VectorLoopRegion->getEntryBasicBlock()->phis())) {
auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&Phi);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 8fbf6793e90c5..89272fcfd05dc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -292,12 +292,11 @@ struct VPlanTransforms {
/// regions until no improvements are remaining.
static void createAndOptimizeReplicateRegions(VPlan &Plan);
- /// Replace (ICMP_ULE, wide canonical IV, backedge-taken-count) checks with an
- /// (active-lane-mask recipe, wide canonical IV, trip-count). If \p
- /// UseActiveLaneMaskForControlFlow is true, introduce an
- /// VPActiveLaneMaskPHIRecipe.
- static void addActiveLaneMask(VPlan &Plan,
- bool UseActiveLaneMaskForControlFlow);
+ /// Materialize the abstract header mask of the loop region into concrete
+ /// recipes: an active-lane-mask if \p UseActiveLaneMask (with a PHI if \p
+ /// UseActiveLaneMaskForControlFlow), else (WideCanonicalIV icmp ule BTC).
+ static void materializeHeaderMask(VPlan &Plan, bool UseActiveLaneMask,
+ bool UseActiveLaneMaskForControlFlow);
/// Insert truncates and extends for any truncated recipe. Redundant casts
/// will be folded later.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 812a63fd830a7..dd7fa1a5bb6cc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -57,43 +57,6 @@ VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr) {
return Expanded;
}
-bool vputils::isHeaderMask(const VPValue *V, const VPlan &Plan) {
- if (isa<VPActiveLaneMaskPHIRecipe>(V))
- return true;
-
- VPValue *A, *B;
-
- auto m_CanonicalScalarIVSteps = m_ScalarIVSteps(
- m_CombineOr(m_CanonicalIV(),
- m_DerivedIV(m_ZeroInt(), m_CanonicalIV(), m_One())),
- m_One(), m_Specific(&Plan.getVF()));
-
- // A wide canonical IV is either an explicit VPWidenCanonicalIVRecipe or a
- // canonical VPWidenIntOrFpInductionRecipe.
- auto m_WideCanonicalIV =
- m_CombineOr(m_Isa<VPWidenCanonicalIVRecipe>(), m_CanonicalWidenIV());
-
- if (match(V, m_ActiveLaneMask(m_VPValue(A), m_VPValue(B), m_One())))
- return B == Plan.getTripCount() &&
- (match(A, m_CanonicalScalarIVSteps) || match(A, m_WideCanonicalIV));
-
- // For scalar plans, the header mask uses the scalar steps.
- if (match(V, m_ICmp(m_CanonicalScalarIVSteps,
- m_Specific(Plan.getBackedgeTakenCount())))) {
- assert(Plan.hasScalarVFOnly() &&
- "Non-scalar VF using scalar IV steps for header mask?");
- return true;
- }
-
- auto MaskMatch = m_ICmp(m_VPValue(A), m_VPValue(B));
- if (match(V, m_CombineOr(MaskMatch, m_Reverse(MaskMatch))))
- return match(A, m_WideCanonicalIV) && B == Plan.getBackedgeTakenCount();
-
- return match(
- V, m_c_BinaryAnd(m_VPValue(),
- m_VPInstruction<VPInstruction::IncomingAliasMask>()));
-}
-
/// Returns true if \p R propagates poison from any operand to its result.
static bool propagatesPoisonFromRecipeOp(const VPRecipeBase *R) {
return TypeSwitch<const VPRecipeBase *, bool>(R)
@@ -420,7 +383,9 @@ static bool preservesUniformity(unsigned Opcode) {
}
bool vputils::isSingleScalar(const VPValue *VPV) {
- // Live-in, symbolic and region-values represent single-scalar values.
+ // Live-in, symbolic and canonical-IV region values are single-scalar.
+ if (auto *RV = dyn_cast<VPRegionValue>(VPV))
+ return RV == RV->getDefiningRegion()->getCanonicalIV();
if (isa<VPIRValue, VPSymbolicValue>(VPV))
return true;
@@ -457,7 +422,9 @@ bool vputils::isSingleScalar(const VPValue *VPV) {
}
bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
- // Live-ins and region values are uniform.
+ // Live-ins, symbolic and canonical-IV region values are uniform.
+ if (auto *RV = dyn_cast<VPRegionValue>(V))
+ return RV == RV->getDefiningRegion()->getCanonicalIV();
if (isa<VPIRValue, VPSymbolicValue>(V))
return true;
@@ -544,60 +511,6 @@ bool vputils::cannotHoistOrSinkRecipe(const VPRecipeBase &R, bool Sinking) {
return RepR && RepR->getOpcode() == Instruction::Alloca;
}
-VPSingleDefRecipe *vputils::findHeaderMask(VPlan &Plan) {
- if (VPValue *AliasMask = findIncomingAliasMask(Plan)) {
- assert(match(AliasMask->getSingleUser(),
- m_c_BinaryAnd(m_VPValue(), m_Specific(AliasMask))) &&
- "AliasMask must only be used with the original header mask");
- return cast<VPSingleDefRecipe>(AliasMask->getSingleUser());
- }
-
- VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
- SmallVector<VPValue *> WideCanonicalIVs;
- auto *WideCanonicalIV =
- findUserOf<VPWidenCanonicalIVRecipe>(LoopRegion->getCanonicalIV());
- assert(count_if(LoopRegion->getCanonicalIV()->users(),
- IsaPred<VPWidenCanonicalIVRecipe>) <= 1 &&
- "Must have at most one VPWideCanonicalIVRecipe");
- if (WideCanonicalIV)
- WideCanonicalIVs.push_back(WideCanonicalIV);
-
- // Also include VPWidenIntOrFpInductionRecipes that represent a widened
- // version of the canonical induction.
- VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
- for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
- VPWidenIntOrFpInductionRecipe *WidenIV;
- if (match(&Phi, m_CanonicalWidenIV(WidenIV)))
- WideCanonicalIVs.push_back(WidenIV);
- }
-
- // Walk users of wide canonical IVs and find the single compare of the form
- // (ICMP_ULE, WideCanonicalIV, backedge-taken-count).
- VPSingleDefRecipe *HeaderMask = nullptr;
- for (auto *Wide : WideCanonicalIVs) {
- for (VPUser *U : Wide->users()) {
- auto *VPI = dyn_cast<VPInstruction>(U);
- if (!VPI || !vputils::isHeaderMask(VPI, Plan))
- continue;
-
- assert(VPI->getOperand(0) == Wide &&
- "WidenCanonicalIV must be the first operand of the compare");
- assert(!HeaderMask && "Multiple header masks found?");
- HeaderMask = VPI;
- }
- }
-
- for (VPRecipeBase &R : LoopRegion->getEntryBasicBlock()->phis()) {
- auto *Def = cast<VPSingleDefRecipe>(&R);
- if (vputils::isHeaderMask(Def, Plan)) {
- assert(!HeaderMask && "Multiple header masks found?");
- HeaderMask = Def;
- }
- }
-
- return HeaderMask;
-}
-
SmallVector<VPBasicBlock *>
VPBlockUtils::blocksInSingleSuccessorChainBetween(VPBasicBlock *FirstBB,
VPBasicBlock *LastBB) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 3ebf8e63d5b36..0f1f93771e26b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -55,9 +55,6 @@ bool isAddressSCEVForCost(const SCEV *Addr, ScalarEvolution &SE, const Loop *L);
/// same value for all lanes or only has its first lane used.
bool isSingleScalar(const VPValue *VPV);
-/// Return true if \p V is a header mask in \p Plan.
-bool isHeaderMask(const VPValue *V, const VPlan &Plan);
-
/// Checks if \p V is uniform across all VF lanes and UF parts. It is considered
/// as such if it is either loop invariant (defined outside the vector region)
/// or its operands are known to be uniform across all VFs and UFs (e.g.
@@ -159,14 +156,6 @@ bool isUsedByLoadStoreAddress(const VPValue *V);
/// inserted for predicated reductions or tail folding.
VPInstruction *findComputeReductionResult(VPReductionPHIRecipe *PhiR);
-/// Collect the header mask with the pattern:
-/// (ICMP_ULE, WideCanonicalIV, backedge-taken-count)
-/// Note: If alias masking is enabled this will find:
-/// (AND, HeaderMask, AliasMask)
-/// TODO: Introduce explicit recipe for header-mask instead of searching
-/// the header-mask pattern manually.
-VPSingleDefRecipe *findHeaderMask(VPlan &Plan);
-
/// Finds the incoming alias-mask within the vector preheader.
VPValue *findIncomingAliasMask(const VPlan &Plan);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 362bfe92f573e..68ec18093e810 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -179,8 +179,9 @@ bool VPlanVerifier::verifyLastActiveLaneRecipe(
VPValue *Mask = Op;
VPValue *HeaderMask;
- // Look through any `and`s with a loop_dependence_war_mask, which is always
- // a prefix mask. TODO: Verify the full loop.dependence.mask chain.
+ // Look through any `and`s with the incoming alias mask or a
+ // loop_dependence_war_mask, which are always prefix masks.
+ // TODO: Verify the full loop.dependence.mask chain.
if (match(Op,
m_c_BinaryAnd(
m_VPValue(HeaderMask),
@@ -188,10 +189,15 @@ bool VPlanVerifier::verifyLastActiveLaneRecipe(
m_c_BinaryAnd(
m_Intrinsic<Intrinsic::loop_dependence_war_mask>(),
m_VPValue()),
- m_Intrinsic<Intrinsic::loop_dependence_war_mask>()))))
+ m_Intrinsic<Intrinsic::loop_dependence_war_mask>(),
+ m_VPInstruction<VPInstruction::IncomingAliasMask>()))))
Mask = HeaderMask;
- if (vputils::isHeaderMask(Mask, Plan))
+ // The header mask is a prefix mask. Before being materialized it is the
+ // loop region's abstract header mask; afterwards it is an active lane mask
+ // (an intrinsic or a phi), or the icmp checked below.
+ if (match(Mask, m_HeaderMask()) || isa<VPActiveLaneMaskPHIRecipe>(Mask) ||
+ match(Mask, m_VPInstruction<VPInstruction::ActiveLaneMask>()))
continue;
if (match(Mask, m_ActiveLaneMask(m_VPValue(), m_VPValue(), m_VPValue())))
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index ec5c95fb0b17d..96ba957c56d97 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -477,7 +477,6 @@ define i32 @header_mask_and_invariant_compare(ptr %A, ptr %B, ptr %C, ptr %D, pt
; PRED: [[VECTOR_PH]]:
; PRED-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
; PRED-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 2
-; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
; PRED-NEXT: [[L_A:%.*]] = load i32, ptr [[A]], align 4, !alias.scope [[META3:![0-9]+]]
; PRED-NEXT: [[L_B:%.*]] = load i32, ptr [[B]], align 4, !alias.scope [[META6:![0-9]+]]
; PRED-NEXT: [[OR:%.*]] = or i32 [[L_B]], [[L_A]]
@@ -487,6 +486,7 @@ define i32 @header_mask_and_invariant_compare(ptr %A, ptr %B, ptr %C, ptr %D, pt
; PRED-NEXT: [[C_0:%.*]] = icmp ugt i32 [[L_C]], [[OR]]
; PRED-NEXT: [[BROADCAST_SPLATINSERT28:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[C_0]], i64 0
; PRED-NEXT: [[BROADCAST_SPLAT29:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT28]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
+; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[E]], i64 0
; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
; PRED-NEXT: br label %[[VECTOR_BODY:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fold-tail-low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fold-tail-low-trip-count.ll
index a9915e033fd6a..95d348576d4e1 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fold-tail-low-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fold-tail-low-trip-count.ll
@@ -16,10 +16,10 @@ define void @low_trip_count_small(i32 %x, ptr %dst) {
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ule <4 x i64> <i64 0, i64 1, i64 2, i64 3>, [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[DST]], i64 1
; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[DST]], i64 2
; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[DST]], i64 3
-; CHECK-NEXT: [[TMP5:%.*]] = icmp ule <4 x i64> <i64 0, i64 1, i64 2, i64 3>, [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i64 0
; CHECK-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
@@ -89,10 +89,10 @@ define ptr @low_trip_count_small_with_live_out(i32 %x, ptr %dst) {
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ule <4 x i64> <i64 0, i64 1, i64 2, i64 3>, [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[DST]], i64 1
; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[DST]], i64 2
; CHECK-NEXT: [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[DST]], i64 3
-; CHECK-NEXT: [[TMP6:%.*]] = icmp ule <4 x i64> <i64 0, i64 1, i64 2, i64 3>, [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i64 0
; CHECK-NEXT: br i1 [[TMP7]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
index 6818b3900ab3b..07fc18e46c4fe 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
@@ -600,8 +600,8 @@ define void @forced_scalar_instr(ptr %gep.dst) {
; COMMON: [[VECTOR_BODY]]:
; COMMON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
; COMMON-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ]
-; COMMON-NEXT: [[TMP0:%.*]] = trunc i64 [[INDEX]] to i32
; COMMON-NEXT: [[TMP1:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 4)
+; COMMON-NEXT: [[TMP0:%.*]] = trunc i64 [[INDEX]] to i32
; COMMON-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
; COMMON-NEXT: br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; COMMON: [[PRED_STORE_IF]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
index 3675576d6bbf4..192f1d805f079 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
@@ -133,8 +133,8 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
; PRED-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 3
; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[X]], i64 0
; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
-; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP0]])
; PRED-NEXT: [[TMP6:%.*]] = trunc <vscale x 8 x i32> [[BROADCAST_SPLAT]] to <vscale x 8 x i16>
+; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP0]])
; PRED-NEXT: br label %[[VECTOR_BODY:.*]]
; PRED: [[VECTOR_BODY]]:
; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
index 1938212173b6c..0617cae585993 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/predicated-costs.ll
@@ -66,65 +66,45 @@ define void @test_predicated_load_cast_hint(ptr %dst.1, ptr %dst.2, ptr %src, i8
; CHECK-NEXT: [[CONFLICT_RDX15:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT14]]
; CHECK-NEXT: br i1 [[CONFLICT_RDX15]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 0, i32 [[TMP2]])
-; CHECK-NEXT: [[TMP28:%.*]] = load i8, ptr [[SRC]], align 1, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[TMP28]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP25:%.*]] = zext <4 x i8> [[BROADCAST_SPLAT]] to <4 x i64>
+; CHECK-NEXT: [[TMP27:%.*]] = load i8, ptr [[SRC]], align 1, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i8> poison, i8 [[TMP27]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i8> [[BROADCAST_SPLATINSERT]], <2 x i8> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP25:%.*]] = zext <2 x i8> [[BROADCAST_SPLAT]] to <2 x i64>
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 0, i32 [[TMP2]])
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE22:.*]] ]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE22]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 4, i8 8, i8 12>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE22]] ]
-; CHECK-NEXT: [[TMP26:%.*]] = zext <4 x i8> [[VEC_IND]] to <4 x i64>
-; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK]], i64 0
-; CHECK-NEXT: br i1 [[TMP27]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; CHECK: [[PRED_STORE_IF]]:
-; CHECK-NEXT: [[TMP102:%.*]] = extractelement <4 x i64> [[TMP26]], i64 0
-; CHECK-NEXT: [[TMP103:%.*]] = getelementptr [16 x i64], ptr [[DST_1]], i64 [[TMP102]], i64 [[OFF]]
-; CHECK-NEXT: [[TMP104:%.*]] = extractelement <4 x i64> [[TMP25]], i64 0
-; CHECK-NEXT: [[TMP105:%.*]] = or i64 [[TMP104]], 1
-; CHECK-NEXT: store i64 [[TMP105]], ptr [[TMP103]], align 8, !alias.scope [[META3]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE]]
-; CHECK: [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK]], i64 1
-; CHECK-NEXT: br i1 [[TMP32]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18:.*]]
-; CHECK: [[PRED_STORE_IF17]]:
-; CHECK-NEXT: [[TMP108:%.*]] = extractelement <4 x i64> [[TMP26]], i64 1
-; CHECK-NEXT: [[TMP109:%.*]] = getelementptr [16 x i64], ptr [[DST_1]], i64 [[TMP108]], i64 [[OFF]]
-; CHECK-NEXT: [[TMP110:%.*]] = extractelement <4 x i64> [[TMP25]], i64 1
-; CHECK-NEXT: [[TMP111:%.*]] = or i64 [[TMP110]], 1
-; CHECK-NEXT: store i64 [[TMP111]], ptr [[TMP109]], align 8, !alias.scope [[META3]]
-; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE18]]
-; CHECK: [[PRED_STORE_CONTINUE18]]:
-; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK]], i64 2
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[PRED_STORE_CONTINUE22]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i8> [ <i8 0, i8 4>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE22]] ]
+; CHECK-NEXT: [[TMP26:%.*]] = zext <2 x i8> [[VEC_IND]] to <2 x i64>
+; CHECK-NEXT: [[TMP37:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 0
; CHECK-NEXT: br i1 [[TMP37]], label %[[PRED_STORE_IF19:.*]], label %[[PRED_STORE_CONTINUE20:.*]]
; CHECK: [[PRED_STORE_IF19]]:
-; CHECK-NEXT: [[TMP114:%.*]] = extractelement <4 x i64> [[TMP26]], i64 2
+; CHECK-NEXT: [[TMP114:%.*]] = extractelement <2 x i64> [[TMP26]], i64 0
; CHECK-NEXT: [[TMP115:%.*]] = getelementptr [16 x i64], ptr [[DST_1]], i64 [[TMP114]], i64 [[OFF]]
-; CHECK-NEXT: [[TMP116:%.*]] = extractelement <4 x i64> [[TMP25]], i64 2
+; CHECK-NEXT: [[TMP116:%.*]] = extractelement <2 x i64> [[TMP25]], i64 0
; CHECK-NEXT: [[TMP117:%.*]] = or i64 [[TMP116]], 1
; CHECK-NEXT: store i64 [[TMP117]], ptr [[TMP115]], align 8, !alias.scope [[META3]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE20]]
; CHECK: [[PRED_STORE_CONTINUE20]]:
-; CHECK-NEXT: [[TMP42:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK]], i64 3
+; CHECK-NEXT: [[TMP42:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK]], i64 1
; CHECK-NEXT: br i1 [[TMP42]], label %[[PRED_STORE_IF21:.*]], label %[[PRED_STORE_CONTINUE22]]
; CHECK: [[PRED_STORE_IF21]]:
-; CHECK-NEXT: [[TMP120:%.*]] = extractelement <4 x i64> [[TMP26]], i64 3
+; CHECK-NEXT: [[TMP120:%.*]] = extractelement <2 x i64> [[TMP26]], i64 1
; CHECK-NEXT: [[TMP121:%.*]] = getelementptr [16 x i64], ptr [[DST_1]], i64 [[TMP120]], i64 [[OFF]]
-; CHECK-NEXT: [[TMP122:%.*]] = extractelement <4 x i64> [[TMP25]], i64 3
+; CHECK-NEXT: [[TMP122:%.*]] = extractelement <2 x i64> [[TMP25]], i64 1
; CHECK-NEXT: [[TMP123:%.*]] = or i64 [[TMP122]], 1
; CHECK-NEXT: store i64 [[TMP123]], ptr [[TMP121]], align 8, !alias.scope [[META3]]
; CHECK-NEXT: br label %[[PRED_STORE_CONTINUE22]]
; CHECK: [[PRED_STORE_CONTINUE22]]:
-; CHECK-NEXT: store i8 0, ptr [[DST_2]], align 1, !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]
-; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX_NEXT]], i32 [[TMP2]])
-; CHECK-NEXT: [[TMP47:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 2
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 [[INDEX_NEXT]], i32 [[TMP2]])
+; CHECK-NEXT: [[TMP47:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-NEXT: [[TMP48:%.*]] = xor i1 [[TMP47]], true
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 16)
-; CHECK-NEXT: br i1 [[TMP48]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i8> [[VEC_IND]], splat (i8 8)
+; CHECK-NEXT: br i1 [[TMP48]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: store i8 0, ptr [[DST_2]], align 1, !alias.scope [[META9:![0-9]+]], !noalias [[META11:![0-9]+]]
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP:.*]]
@@ -502,15 +482,15 @@ exit:
; CHECK: [[META2]] = distinct !{[[META2]], !"LVerDomain"}
; CHECK: [[META3]] = !{[[META4:![0-9]+]]}
; CHECK: [[META4]] = distinct !{[[META4]], [[META2]]}
-; CHECK: [[META5]] = !{[[META6:![0-9]+]]}
-; CHECK: [[META6]] = distinct !{[[META6]], [[META2]]}
-; CHECK: [[META7]] = !{[[META1]], [[META4]]}
-; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META9:![0-9]+]], [[META10:![0-9]+]], [[META11:![0-9]+]]}
-; CHECK: [[META9]] = !{!"llvm.loop.mustprogress"}
-; CHECK: [[META10]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META11]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META9]], [[META10]]}
-; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META10]], [[META11]]}
-; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META11]], [[META10]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META6:![0-9]+]], [[META7:![0-9]+]], [[META8:![0-9]+]]}
+; CHECK: [[META6]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[META7]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META8]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[META9]] = !{[[META10:![0-9]+]]}
+; CHECK: [[META10]] = distinct !{[[META10]], [[META2]]}
+; CHECK: [[META11]] = !{[[META1]], [[META4]]}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META6]], [[META7]]}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META7]], [[META8]]}
+; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META8]], [[META7]]}
; CHECK: [[PROF15]] = !{!"branch_weights", i32 0, i32 1}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
index ac69ea10bd1b6..449666e9f47bb 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
@@ -202,7 +202,6 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
; PRED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
; PRED-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
; PRED-NEXT: [[TMP11:%.*]] = add i64 [[Y]], 1
; PRED-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[SRC_1]], i64 [[TMP11]]
; PRED-NEXT: [[TMP13:%.*]] = lshr <vscale x 4 x i32> [[BROADCAST_SPLAT]], splat (i32 1)
@@ -215,6 +214,7 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
; PRED-NEXT: [[TMP18:%.*]] = xor <vscale x 4 x i32> [[TMP17]], splat (i32 1)
; PRED-NEXT: [[TMP19:%.*]] = zext <vscale x 4 x i32> [[TMP18]] to <vscale x 4 x i64>
; PRED-NEXT: [[DOTSPLAT:%.*]] = getelementptr i32, ptr [[SRC_2]], <vscale x 4 x i64> [[TMP19]]
+; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
; PRED-NEXT: [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
; PRED-NEXT: [[TMP23:%.*]] = mul nuw i32 [[TMP22]], 4
; PRED-NEXT: [[TMP24:%.*]] = sub i32 [[TMP23]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
index ce1560e80dad2..1b46ee158117c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
@@ -206,13 +206,13 @@ define void @trunc_store(ptr %dst, ptr %src, i16 %x) #1 {
; PRED-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
; PRED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i16> poison, i16 [[X]], i64 0
; PRED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i16> poison, <vscale x 16 x i32> zeroinitializer
-; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 1000)
; PRED-NEXT: [[TMP4:%.*]] = load i64, ptr [[SRC]], align 8, !alias.scope [[META3:![0-9]+]]
; PRED-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 16 x i64> poison, i64 [[TMP4]], i64 0
; PRED-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 16 x i64> [[BROADCAST_SPLATINSERT2]], <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer
; PRED-NEXT: [[TMP3:%.*]] = trunc <vscale x 16 x i64> [[BROADCAST_SPLAT3]] to <vscale x 16 x i8>
; PRED-NEXT: [[TMP2:%.*]] = trunc <vscale x 16 x i16> [[BROADCAST_SPLAT]] to <vscale x 16 x i8>
; PRED-NEXT: [[TMP5:%.*]] = and <vscale x 16 x i8> [[TMP3]], [[TMP2]]
+; PRED-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 1000)
; PRED-NEXT: br label [[VECTOR_BODY:%.*]]
; PRED: vector.body:
; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/widen-gep-all-indices-invariant.ll b/llvm/test/Transforms/LoopVectorize/AArch64/widen-gep-all-indices-invariant.ll
index 66d229dbaaa19..fbc313eddea8a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/widen-gep-all-indices-invariant.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/widen-gep-all-indices-invariant.ll
@@ -11,11 +11,11 @@ define i32 @gep_with_all_invariant_operands(ptr %src.0, ptr %src.1, i64 %n, i1 %
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[SRC_0]], i64 [[N]]
; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[COND]], ptr [[SRC_1]], ptr [[TMP8]]
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[TMP9]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP0]])
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-saddsatcost.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-saddsatcost.ll
index 8d7a21fe15930..c6f8aae37e92b 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-saddsatcost.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-saddsatcost.ll
@@ -25,10 +25,10 @@ define void @arm_offset_q15(ptr nocapture readonly %pSrc, i16 signext %offset, p
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX]], i32 [[BLOCKSIZE]])
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 1
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PSRC]], i32 [[OFFSET_IDX]]
; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[PDST]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 [[INDEX]], i32 [[BLOCKSIZE]])
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 2 [[NEXT_GEP]], <8 x i1> [[ACTIVE_LANE_MASK]], <8 x i16> poison)
; CHECK-NEXT: [[TMP0:%.*]] = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> [[WIDE_MASKED_LOAD]], <8 x i16> [[BROADCAST_SPLAT]])
; CHECK-NEXT: call void @llvm.masked.store.v8i16.p0(<8 x i16> [[TMP0]], ptr align 2 [[NEXT_GEP2]], <8 x i1> [[ACTIVE_LANE_MASK]])
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
index 3fbfa32b6fccf..b9363193d2306 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
@@ -29,10 +29,10 @@ define void @sgt_loopguard(ptr noalias nocapture readonly %a, ptr noalias nocapt
; COMMON-NEXT: br label %[[VECTOR_BODY:.*]]
; COMMON: [[VECTOR_BODY]]:
; COMMON-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
; COMMON-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i32 [[INDEX]]
; COMMON-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
; COMMON-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i32 [[INDEX]]
-; COMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
; COMMON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP2]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; COMMON-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; COMMON-NEXT: [[TMP0:%.*]] = add <16 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
@@ -94,10 +94,10 @@ define void @sgt_no_loopguard(ptr noalias nocapture readonly %a, ptr noalias noc
; COMMON-NEXT: br label %[[VECTOR_BODY:.*]]
; COMMON: [[VECTOR_BODY]]:
; COMMON-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[TMP1]])
; COMMON-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i32 [[INDEX]]
; COMMON-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
; COMMON-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i32 [[INDEX]]
-; COMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[TMP1]])
; COMMON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP2]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; COMMON-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; COMMON-NEXT: [[TMP2:%.*]] = add <16 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
@@ -157,10 +157,10 @@ define void @sgt_extra_use_cmp(ptr noalias nocapture readonly %a, ptr noalias no
; COMMON: [[VECTOR_BODY]]:
; COMMON-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMMON-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[TMP1]])
; COMMON-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i32 [[INDEX]]
; COMMON-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
; COMMON-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i32 [[INDEX]]
-; COMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[TMP1]])
; COMMON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[NEXT_GEP2]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison)
; COMMON-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[NEXT_GEP1]], <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison)
; COMMON-NEXT: [[TMP3:%.*]] = add <4 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
@@ -219,10 +219,10 @@ define void @sgt_const_tripcount(ptr noalias nocapture readonly %a, ptr noalias
; COMMON-NEXT: br label %[[VECTOR_BODY:.*]]
; COMMON: [[VECTOR_BODY]]:
; COMMON-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 2049)
; COMMON-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i32 [[INDEX]]
; COMMON-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
; COMMON-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i32 [[INDEX]]
-; COMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 2049)
; COMMON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP2]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; COMMON-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; COMMON-NEXT: [[TMP0:%.*]] = add <16 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
@@ -335,10 +335,10 @@ define void @sgt_step_minus_two(ptr noalias nocapture readonly %a, ptr noalias n
; COMMON-NEXT: br label %[[VECTOR_BODY:.*]]
; COMMON: [[VECTOR_BODY]]:
; COMMON-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[TMP1]])
; COMMON-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i32 [[INDEX]]
; COMMON-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
; COMMON-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i32 [[INDEX]]
-; COMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[TMP1]])
; COMMON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP2]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; COMMON-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; COMMON-NEXT: [[TMP2:%.*]] = add <16 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
@@ -456,10 +456,10 @@ define void @icmp_eq(ptr noalias nocapture readonly %A, ptr noalias nocapture re
; COMMON-NEXT: br label %[[VECTOR_BODY:.*]]
; COMMON: [[VECTOR_BODY]]:
; COMMON-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
; COMMON-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[C]], i32 [[INDEX]]
; COMMON-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i32 [[INDEX]]
; COMMON-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[A]], i32 [[INDEX]]
-; COMMON-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
; COMMON-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP2]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; COMMON-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP1]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; COMMON-NEXT: [[TMP0:%.*]] = add <16 x i8> [[WIDE_MASKED_LOAD3]], [[WIDE_MASKED_LOAD]]
@@ -567,8 +567,8 @@ define void @sgt_for_loop(ptr noalias nocapture readonly %a, ptr noalias nocaptu
; CHECK-PREFER-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-PREFER: [[VECTOR_BODY]]:
; CHECK-PREFER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-PREFER-NEXT: [[TMP0:%.*]] = sub i32 [[N]], [[INDEX]]
; CHECK-PREFER-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[N]])
+; CHECK-PREFER-NEXT: [[TMP0:%.*]] = sub i32 [[N]], [[INDEX]]
; CHECK-PREFER-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP0]]
; CHECK-PREFER-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 -15
; CHECK-PREFER-NEXT: [[REVERSE:%.*]] = shufflevector <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i1> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
@@ -709,8 +709,8 @@ define void @sgt_for_loop_i64(ptr noalias nocapture readonly %a, ptr noalias noc
; CHECK-PREFER-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-PREFER: [[VECTOR_BODY]]:
; CHECK-PREFER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-PREFER-NEXT: [[TMP0:%.*]] = sub i64 [[CONV16]], [[INDEX]]
; CHECK-PREFER-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 [[INDEX]], i64 [[CONV16]])
+; CHECK-PREFER-NEXT: [[TMP0:%.*]] = sub i64 [[CONV16]], [[INDEX]]
; CHECK-PREFER-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
; CHECK-PREFER-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[TMP1]]
; CHECK-PREFER-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[TMP2]], i32 -15
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/force-target-instruction-cost.ll
index 9cf0eaa8d08a9..bb178bec5aeb1 100644
--- a/llvm/test/Transforms/LoopVectorize/SystemZ/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/force-target-instruction-cost.ll
@@ -13,8 +13,8 @@ define void @test_scalar_steps_target_instruction_cost(ptr %dst) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i8> [ <i8 0, i8 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE2]] ]
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <2 x i8> [[VEC_IND]], splat (i8 8)
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i1> [[TMP0]], i64 0
; CHECK-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/predicated-first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/predicated-first-order-recurrence.ll
index 4eecbd77b0398..aa2926e55a8d2 100644
--- a/llvm/test/Transforms/LoopVectorize/SystemZ/predicated-first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/predicated-first-order-recurrence.ll
@@ -20,8 +20,8 @@ define void @func_21() {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE4:%.*]] ]
; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i32> [ <i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[PRED_STORE_CONTINUE4]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i8> [ <i8 0, i8 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE4]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <2 x i8> [[VEC_IND]], splat (i8 4)
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i64 0
; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
; CHECK: pred.load.if:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/X86/vplan-vp-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/VPlan/X86/vplan-vp-intrinsics.ll
index 48370f9d38ce9..f1f6a2c1c80da 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/X86/vplan-vp-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/X86/vplan-vp-intrinsics.ll
@@ -22,9 +22,9 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
; IF-EVL-NEXT: vp<[[IV:%[0-9]+]]> = CANONICAL-IV
; IF-EVL-EMPTY:
; IF-EVL-NEXT: vector.body:
-; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>, vp<[[VF]]>
; IF-EVL-NEXT: EMIT vp<[[VIV:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[IV]]>
; IF-EVL-NEXT: EMIT vp<[[MASK:%[0-9]+]]> = icmp ule vp<[[VIV]]>, vp<[[BETC]]>
+; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>, vp<[[VF]]>
; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>
; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = load vp<[[PTR1]]>, vp<[[MASK]]>
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
index 4251189815f49..89bb9899d104f 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/conditional-scalar-assignment-vplan.ll
@@ -93,19 +93,19 @@ define i32 @simple_csa_int_select(i64 %N, ptr %data, i32 %a) {
; CHECK-TF-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: vector.body:
-; CHECK-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP13:%[0-9]+]]>
-; CHECK-TF-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP12:%[0-9]+]]>, vector.body ]
-; CHECK-TF-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
+; CHECK-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<-1>, vp<[[VP14:%[0-9]+]]>
+; CHECK-TF-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP13:%[0-9]+]]>, vector.body ]
; CHECK-TF-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
; CHECK-TF-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
-; CHECK-TF-NEXT: CLONE ir<%ld.addr> = getelementptr inbounds ir<%data>, vp<[[VP6]]>
-; CHECK-TF-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds ir<%ld.addr>, ir<1>
-; CHECK-TF-NEXT: WIDEN ir<%ld> = load vp<[[VP9]]>, vp<[[VP8]]>
+; CHECK-TF-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
+; CHECK-TF-NEXT: CLONE ir<%ld.addr> = getelementptr inbounds ir<%data>, vp<[[VP9]]>
+; CHECK-TF-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%ld.addr>, ir<1>
+; CHECK-TF-NEXT: WIDEN ir<%ld> = load vp<[[VP10]]>, vp<[[VP8]]>
; CHECK-TF-NEXT: WIDEN ir<%select.cmp> = icmp slt ir<%a>, ir<%ld>
-; CHECK-TF-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = logical-and vp<[[VP8]]>, ir<%select.cmp>
-; CHECK-TF-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = any-of vp<[[VP10]]>
-; CHECK-TF-NEXT: EMIT vp<[[VP12]]> = select vp<[[VP11]]>, vp<[[VP10]]>, vp<[[VP5]]>
-; CHECK-TF-NEXT: EMIT vp<[[VP13]]> = select vp<[[VP11]]>, ir<%ld>, ir<%data.phi>
+; CHECK-TF-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = logical-and vp<[[VP8]]>, ir<%select.cmp>
+; CHECK-TF-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = any-of vp<[[VP11]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP13]]> = select vp<[[VP12]]>, vp<[[VP11]]>, vp<[[VP6]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP14]]> = select vp<[[VP12]]>, ir<%ld>, ir<%data.phi>
; CHECK-TF-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP1]]>
; CHECK-TF-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-TF-NEXT: No successors
@@ -113,11 +113,11 @@ define i32 @simple_csa_int_select(i64 %N, ptr %data, i32 %a) {
; CHECK-TF-NEXT: Successor(s): middle.block
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: middle.block:
-; CHECK-TF-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP13]]>, vp<[[VP12]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = extract-last-active ir<-1>, vp<[[VP14]]>, vp<[[VP13]]>
; CHECK-TF-NEXT: Successor(s): ir-bb<exit>
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: ir-bb<exit>:
-; CHECK-TF-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %loop ] (extra operand: vp<[[VP15]]> from middle.block)
+; CHECK-TF-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %loop ] (extra operand: vp<[[VP16]]> from middle.block)
; CHECK-TF-NEXT: No successors
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: scalar.ph:
@@ -256,22 +256,22 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val
; CHECK-TF-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: vector.body:
-; CHECK-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<%default_val>, vp<[[VP14:%[0-9]+]]>
-; CHECK-TF-NEXT: WIDEN-PHI vp<[[VP5:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP13:%[0-9]+]]>, vector.body ]
-; CHECK-TF-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
+; CHECK-TF-NEXT: WIDEN-REDUCTION-PHI ir<%data.phi> = phi (find-last) ir<%default_val>, vp<[[VP15:%[0-9]+]]>
+; CHECK-TF-NEXT: WIDEN-PHI vp<[[VP6:%[0-9]+]]> = phi [ ir<false>, vector.ph ], [ vp<[[VP14:%[0-9]+]]>, vector.body ]
; CHECK-TF-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
; CHECK-TF-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
-; CHECK-TF-NEXT: CLONE ir<%a.addr> = getelementptr inbounds nuw ir<%a>, vp<[[VP6]]>
-; CHECK-TF-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds nuw ir<%a.addr>, ir<1>
-; CHECK-TF-NEXT: WIDEN ir<%ld.a> = load vp<[[VP9]]>, vp<[[VP8]]>
+; CHECK-TF-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
+; CHECK-TF-NEXT: CLONE ir<%a.addr> = getelementptr inbounds nuw ir<%a>, vp<[[VP9]]>
+; CHECK-TF-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds nuw ir<%a.addr>, ir<1>
+; CHECK-TF-NEXT: WIDEN ir<%ld.a> = load vp<[[VP10]]>, vp<[[VP8]]>
; CHECK-TF-NEXT: WIDEN ir<%if.cond> = icmp sgt ir<%ld.a>, ir<%threshold>
-; CHECK-TF-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = logical-and vp<[[VP8]]>, ir<%if.cond>
-; CHECK-TF-NEXT: CLONE ir<%b.addr> = getelementptr ir<%b>, vp<[[VP6]]>
-; CHECK-TF-NEXT: vp<[[VP11:%[0-9]+]]> = vector-pointer ir<%b.addr>, ir<1>
-; CHECK-TF-NEXT: WIDEN ir<%ld.b> = load vp<[[VP11]]>, vp<[[VP10]]>
-; CHECK-TF-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = any-of vp<[[VP10]]>
-; CHECK-TF-NEXT: EMIT vp<[[VP13]]> = select vp<[[VP12]]>, vp<[[VP10]]>, vp<[[VP5]]>
-; CHECK-TF-NEXT: EMIT vp<[[VP14]]> = select vp<[[VP12]]>, ir<%ld.b>, ir<%data.phi>
+; CHECK-TF-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = logical-and vp<[[VP8]]>, ir<%if.cond>
+; CHECK-TF-NEXT: CLONE ir<%b.addr> = getelementptr ir<%b>, vp<[[VP9]]>
+; CHECK-TF-NEXT: vp<[[VP12:%[0-9]+]]> = vector-pointer ir<%b.addr>, ir<1>
+; CHECK-TF-NEXT: WIDEN ir<%ld.b> = load vp<[[VP12]]>, vp<[[VP11]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = any-of vp<[[VP11]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP14]]> = select vp<[[VP13]]>, vp<[[VP11]]>, vp<[[VP6]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP15]]> = select vp<[[VP13]]>, ir<%ld.b>, ir<%data.phi>
; CHECK-TF-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP1]]>
; CHECK-TF-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-TF-NEXT: No successors
@@ -279,11 +279,11 @@ define i32 @simple_csa_int_load(ptr noalias %a, ptr noalias %b, i32 %default_val
; CHECK-TF-NEXT: Successor(s): middle.block
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: middle.block:
-; CHECK-TF-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = extract-last-active ir<%default_val>, vp<[[VP14]]>, vp<[[VP13]]>
+; CHECK-TF-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = extract-last-active ir<%default_val>, vp<[[VP15]]>, vp<[[VP14]]>
; CHECK-TF-NEXT: Successor(s): ir-bb<exit>
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: ir-bb<exit>:
-; CHECK-TF-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %latch ] (extra operand: vp<[[VP16]]> from middle.block)
+; CHECK-TF-NEXT: IR %select.data.lcssa = phi i32 [ %select.data, %latch ] (extra operand: vp<[[VP17]]> from middle.block)
; CHECK-TF-NEXT: No successors
; CHECK-TF-EMPTY:
; CHECK-TF-NEXT: scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
index 941ea1bd97b67..782c7830f9561 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/first-order-recurrence-sink-replicate-region.ll
@@ -25,9 +25,9 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv>
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP3]]>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -36,19 +36,19 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
-; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[VP5]]>
+; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[VP8]]>
; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V)
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP8:%[0-9]+]]> = ir<%lv>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP9:%[0-9]+]]> = ir<%lv>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:
; CHECK-NEXT: loop.0:
-; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext vp<[[VP8]]> to i32
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = first-order splice ir<%0>, ir<%conv>
+; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext vp<[[VP9]]> to i32
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = first-order splice ir<%0>, ir<%conv>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -57,8 +57,8 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[VP9]]>, ir<%x>
-; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[VP5]]>
+; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[VP10]]>, ir<%x>
+; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[VP8]]>
; CHECK-NEXT: REPLICATE ir<%add> = add ir<%conv>, ir<%rem>
; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.dst>
; CHECK-NEXT: Successor(s): pred.store.continue
@@ -141,22 +141,22 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr, i32 %z) optsize {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = icmp ule vp<[[VP5]]>, vp<[[VP3]]>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = first-order splice ir<%recur>, ir<%recur.next>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP3]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = first-order splice ir<%recur>, ir<%recur.next>
; CHECK-NEXT: WIDEN ir<%cond> = icmp eq ir<%iv>, ir<%z>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP6]]>, ir<%cond>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP7]]>, ir<%cond>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP8]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[VP7]]>, ir<%x>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[VP9]]>
+; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[VP8]]>, ir<%x>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[VP10]]>
; CHECK-NEXT: REPLICATE ir<%add> = add ir<%rem>, ir<%recur.next>
; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep>
; CHECK-NEXT: Successor(s): pred.store.continue
@@ -237,28 +237,28 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%and.red> = phi (and) vp<[[VP3]]>, ir<%and.red.next>
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = icmp ule vp<[[VP5]]>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = first-order splice ir<%recur>, ir<%recur.next>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = first-order splice ir<%recur>, ir<%recur.next>
; CHECK-NEXT: Successor(s): pred.srem
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.srem: {
; CHECK-NEXT: pred.srem.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP6]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP7]]>
; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.srem.if:
-; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[VP7]]>, ir<%x> (S->V)
+; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[VP8]]>, ir<%x> (S->V)
; CHECK-NEXT: Successor(s): pred.srem.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.srem.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP8:%[0-9]+]]> = ir<%rem>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP9:%[0-9]+]]> = ir<%rem>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:
; CHECK-NEXT: loop.0:
-; CHECK-NEXT: WIDEN ir<%add> = add vp<[[VP8]]>, ir<%recur.next>
+; CHECK-NEXT: WIDEN ir<%add> = add vp<[[VP9]]>, ir<%recur.next>
; CHECK-NEXT: WIDEN ir<%and.red.next> = and ir<%and.red>, ir<%add>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
@@ -267,12 +267,12 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = select vp<[[VP6]]>, ir<%and.red.next>, ir<%and.red>
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (and) vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = select vp<[[VP7]]>, ir<%and.red.next>, ir<%and.red>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = compute-reduction-result (and) vp<[[VP11]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %res = phi i32 [ %and.red.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
+; CHECK-NEXT: IR %res = phi i32 [ %and.red.next, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -332,10 +332,10 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%0> = phi ir<0>, ir<%conv>
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP3]]>
-; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[VP5]]>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[VP8]]>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -348,14 +348,14 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP8:%[0-9]+]]> = ir<%lv>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP9:%[0-9]+]]> = ir<%lv>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:
; CHECK-NEXT: loop.0:
-; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext vp<[[VP8]]> to i32
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = first-order splice ir<%0>, ir<%conv>
+; CHECK-NEXT: WIDEN-CAST ir<%conv> = sext vp<[[VP9]]> to i32
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = first-order splice ir<%0>, ir<%conv>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -364,20 +364,20 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
-; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[VP9]]>, ir<%x> (S->V)
+; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[VP10]]>, ir<%x> (S->V)
; CHECK-NEXT: REPLICATE ir<%lv.2> = load ir<%gep> (S->V)
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP10:%[0-9]+]]> = ir<%rem>
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP11:%[0-9]+]]> = ir<%lv.2>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP11:%[0-9]+]]> = ir<%rem>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP12:%[0-9]+]]> = ir<%lv.2>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.2
; CHECK-EMPTY:
; CHECK-NEXT: loop.2:
-; CHECK-NEXT: WIDEN ir<%add.1> = add ir<%conv>, vp<[[VP10]]>
-; CHECK-NEXT: WIDEN-CAST ir<%conv.lv.2> = sext vp<[[VP11]]> to i32
+; CHECK-NEXT: WIDEN ir<%add.1> = add ir<%conv>, vp<[[VP11]]>
+; CHECK-NEXT: WIDEN-CAST ir<%conv.lv.2> = sext vp<[[VP12]]> to i32
; CHECK-NEXT: WIDEN ir<%add> = add ir<%add.1>, ir<%conv.lv.2>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
@@ -387,7 +387,7 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[VP5]]>
+; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[VP8]]>
; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.dst>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
@@ -476,23 +476,23 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP3]]>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = first-order splice ir<%recur>, ir<%recur.next>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = first-order splice ir<%recur>, ir<%recur.next>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP7]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP8]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[VP8]]>, ir<%x>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[VP9]]>, ir<%x>
; CHECK-NEXT: REPLICATE ir<%rem.div> = sdiv ir<20>, ir<%rem>
-; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[VP9]]>
+; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[VP10]]>
; CHECK-NEXT: REPLICATE store ir<%rem.div>, ir<%gep>
-; CHECK-NEXT: REPLICATE ir<%gep.2> = getelementptr ir<%dst.2>, vp<[[VP9]]>
+; CHECK-NEXT: REPLICATE ir<%gep.2> = getelementptr ir<%dst.2>, vp<[[VP10]]>
; CHECK-NEXT: REPLICATE store ir<%rem.div>, ir<%gep.2>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
@@ -572,11 +572,11 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%.pn> = phi ir<0>, ir<%l>
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<2> + vp<[[VP4]]> * ir<1>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP3]]>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = DERIVED-IV ir<2> + vp<[[VP4]]> * ir<1>
; CHECK-NEXT: CLONE ir<%l> = load ir<%src>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = first-order splice ir<%.pn>, ir<%l>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = first-order splice ir<%.pn>, ir<%l>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -585,9 +585,9 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%val> = sdiv vp<[[VP8]]>, ir<%x>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[VP9]]>
+; CHECK-NEXT: REPLICATE ir<%val> = sdiv vp<[[VP9]]>, ir<%x>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = SCALAR-STEPS vp<[[VP8]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr ir<%dst>, vp<[[VP10]]>
; CHECK-NEXT: REPLICATE store ir<%val>, ir<%gep.dst>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll b/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
index c5623e3de4b86..e4bada3c3a4c9 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
@@ -7,7 +7,6 @@ define i32 @live_out(ptr noalias %p, i32 %n) {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
-; CHECK-NEXT: Live-in vp<[[VP3:%[0-9]+]]> = backedge-taken count
; CHECK-NEXT: Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
@@ -17,13 +16,12 @@ define i32 @live_out(ptr noalias %p, i32 %n) {
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = HEADER-MASK
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = icmp ule vp<[[VP5]]>, vp<[[VP3]]>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP6]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP4]]>
; CHECK-NEXT: Successor(s): vector.body.split, vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.body.split:
@@ -36,29 +34,29 @@ define i32 @live_out(ptr noalias %p, i32 %n) {
; CHECK-NEXT: Successor(s): vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.latch:
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP8:%[0-9]+]]> = phi [ ir<%y>, vector.body.split ], [ ir<poison>, vector.body ]
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<%y>, vector.body.split ], [ ir<poison>, vector.body ]
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-part vp<[[VP8]]>
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-last-lane vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = extract-last-part vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = extract-last-lane vp<[[VP9]]>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = last-active-lane vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-lane vp<[[VP13]]>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = last-active-lane vp<[[VP4]]>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-lane vp<[[VP11]]>, vp<[[VP6]]>
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %y.lcssa = phi i32 [ %y, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
+; CHECK-NEXT: IR %y.lcssa = phi i32 [ %y, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP8]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
@@ -95,7 +93,6 @@ define i32 @conditional_live_out(ptr noalias %p, i32 %n, i1 %c) {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
-; CHECK-NEXT: Live-in vp<[[VP3:%[0-9]+]]> = backedge-taken count
; CHECK-NEXT: Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
@@ -105,13 +102,12 @@ define i32 @conditional_live_out(ptr noalias %p, i32 %n, i1 %c) {
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = HEADER-MASK
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = icmp ule vp<[[VP5]]>, vp<[[VP3]]>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP6]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP4]]>
; CHECK-NEXT: Successor(s): vector.body.split, vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.body.split:
@@ -132,29 +128,29 @@ define i32 @conditional_live_out(ptr noalias %p, i32 %n, i1 %c) {
; CHECK-NEXT: Successor(s): vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.latch:
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP8:%[0-9]+]]> = phi [ ir<%phi>, latch ], [ ir<poison>, vector.body ]
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6:%[0-9]+]]> = phi [ ir<%phi>, latch ], [ ir<poison>, vector.body ]
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-part vp<[[VP8]]>
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-last-lane vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = extract-last-part vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = extract-last-lane vp<[[VP9]]>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = last-active-lane vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-lane vp<[[VP13]]>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = last-active-lane vp<[[VP4]]>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-lane vp<[[VP11]]>, vp<[[VP6]]>
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %phi.lcssa = phi i32 [ %phi, %latch ] (extra operand: vp<[[VP14]]> from middle.block)
+; CHECK-NEXT: IR %phi.lcssa = phi i32 [ %phi, %latch ] (extra operand: vp<[[VP12]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP8]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
@@ -192,7 +188,6 @@ define void @header_unconditional_branch(ptr noalias %p, i32 %n) {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
-; CHECK-NEXT: Live-in vp<[[VP3:%[0-9]+]]> = backedge-taken count
; CHECK-NEXT: Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
@@ -202,13 +197,12 @@ define void @header_unconditional_branch(ptr noalias %p, i32 %n) {
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = HEADER-MASK
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = icmp ule vp<[[VP5]]>, vp<[[VP3]]>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP6]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP4]]>
; CHECK-NEXT: Successor(s): vector.body.split, vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.body.split:
@@ -220,14 +214,14 @@ define void @header_unconditional_branch(ptr noalias %p, i32 %n) {
; CHECK-NEXT: Successor(s): vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.latch:
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -236,7 +230,7 @@ define void @header_unconditional_branch(ptr noalias %p, i32 %n) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP7]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
@@ -266,7 +260,6 @@ define i32 @reduction(ptr noalias %p, i32 %n) {
; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
-; CHECK-NEXT: Live-in vp<[[VP3:%[0-9]+]]> = backedge-taken count
; CHECK-NEXT: Live-in ir<%n> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
@@ -276,14 +269,13 @@ define i32 @reduction(ptr noalias %p, i32 %n) {
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = HEADER-MASK
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) ir<0>, vp<[[VP8:%[0-9]+]]>
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = icmp ule vp<[[VP5]]>, vp<[[VP3]]>
-; CHECK-NEXT: EMIT branch-on-cond vp<[[VP6]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) ir<0>, vp<[[VP6:%[0-9]+]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP4]]>
; CHECK-NEXT: Successor(s): vector.body.split, vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.body.split:
@@ -295,34 +287,34 @@ define i32 @reduction(ptr noalias %p, i32 %n) {
; CHECK-NEXT: Successor(s): vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.latch:
-; CHECK-NEXT: EMIT-SCALAR vp<[[VP8]]> = phi [ ir<%rdx.next>, vector.body.split ], [ ir<%rdx>, vector.body ]
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP6]]> = phi [ ir<%rdx.next>, vector.body.split ], [ ir<%rdx>, vector.body ]
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-part vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = extract-last-part vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = extract-last-lane vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-part vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = extract-last-lane vp<[[VP11]]>
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-last-part vp<[[VP8]]>
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-last-lane vp<[[VP13]]>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
-; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = last-active-lane vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = extract-lane vp<[[VP15]]>, vp<[[VP8]]>
-; CHECK-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = last-active-lane vp<[[VP6]]>
-; CHECK-NEXT: EMIT vp<[[VP18:%[0-9]+]]> = extract-lane vp<[[VP17]]>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = last-active-lane vp<[[VP4]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-lane vp<[[VP13]]>, vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = last-active-lane vp<[[VP4]]>
+; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = extract-lane vp<[[VP15]]>, vp<[[VP6]]>
; CHECK-NEXT: EMIT branch-on-cond ir<true>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %rdx.next.lcssa = phi i32 [ %rdx.next, %loop ] (extra operand: vp<[[VP18]]> from middle.block)
+; CHECK-NEXT: IR %rdx.next.lcssa = phi i32 [ %rdx.next, %loop ] (extra operand: vp<[[VP16]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP16]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP8]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.merge.rdx> = phi [ vp<[[VP14]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index 14590ff15c1d0..d09f71e11a4c2 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -59,6 +59,7 @@
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] licm
; CHECK-AFTER: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::optimize
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::narrowInterleaveGroups
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::materializeHeaderMask
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] printOptimizedVPlan
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::addMinimumIterationCheck
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::replaceWideCanonicalIVWithWideIV
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll
index bea28d2834cfa..18d7cb0760fa0 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-alias-mask.ll
@@ -8,7 +8,6 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; INITIAL-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
; INITIAL-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
; INITIAL-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
-; INITIAL-NEXT: Live-in vp<[[VP3:%[0-9]+]]> = backedge-taken count
; INITIAL-NEXT: Live-in ir<%n> = original trip-count
; INITIAL-EMPTY:
; INITIAL-NEXT: ir-bb<entry>:
@@ -19,32 +18,31 @@ define void @alias_mask(ptr noalias %a, ptr %b, ptr %c, i64 %n) {
; INITIAL-NEXT: Successor(s): vector loop
; INITIAL-EMPTY:
; INITIAL-NEXT: <x1> vector loop: {
-; INITIAL-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; INITIAL-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; INITIAL-NEXT: vp<[[VP4:%[0-9]+]]> = HEADER-MASK
; INITIAL-EMPTY:
; INITIAL-NEXT: vector.body:
; INITIAL-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
-; INITIAL-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
-; INITIAL-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = icmp ule vp<[[VP5]]>, vp<[[VP3]]>
-; INITIAL-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = and vp<[[VP6]]>, vp<%incoming.alias.mask>
+; INITIAL-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = and vp<[[VP4]]>, vp<%incoming.alias.mask>
; INITIAL-NEXT: Successor(s): vector.body.split
; INITIAL-EMPTY:
; INITIAL-NEXT: vector.body.split:
; INITIAL-NEXT: CLONE ir<%ptr.a> = getelementptr inbounds ir<%a>, ir<%iv>
-; INITIAL-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds ir<%ptr.a>, ir<1>
-; INITIAL-NEXT: WIDEN ir<%ld.a> = load vp<[[VP8]]>, vp<[[VP7]]>
+; INITIAL-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer inbounds ir<%ptr.a>, ir<1>
+; INITIAL-NEXT: WIDEN ir<%ld.a> = load vp<[[VP6]]>, vp<[[VP5]]>
; INITIAL-NEXT: CLONE ir<%ptr.b> = getelementptr inbounds ir<%b>, ir<%iv>
-; INITIAL-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds ir<%ptr.b>, ir<1>
-; INITIAL-NEXT: WIDEN ir<%ld.b> = load vp<[[VP9]]>, vp<[[VP7]]>
+; INITIAL-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds ir<%ptr.b>, ir<1>
+; INITIAL-NEXT: WIDEN ir<%ld.b> = load vp<[[VP7]]>, vp<[[VP5]]>
; INITIAL-NEXT: WIDEN ir<%add> = add ir<%ld.b>, ir<%ld.a>
; INITIAL-NEXT: CLONE ir<%ptr.c> = getelementptr inbounds ir<%c>, ir<%iv>
-; INITIAL-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%ptr.c>, ir<1>
-; INITIAL-NEXT: WIDEN store vp<[[VP10]]>, ir<%add>, vp<[[VP7]]>
+; INITIAL-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds ir<%ptr.c>, ir<1>
+; INITIAL-NEXT: WIDEN store vp<[[VP8]]>, ir<%add>, vp<[[VP5]]>
; INITIAL-NEXT: CLONE ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
; INITIAL-NEXT: CLONE ir<%exitcond.not> = icmp eq ir<%iv.next>, ir<%n>
; INITIAL-NEXT: Successor(s): vector.latch
; INITIAL-EMPTY:
; INITIAL-NEXT: vector.latch:
-; INITIAL-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP1]]>
+; INITIAL-NEXT: EMIT vp<%index.next> = add vp<[[VP3]]>, vp<[[VP1]]>
; INITIAL-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; INITIAL-NEXT: No successors
; INITIAL-NEXT: }
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions-tail-folded.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions-tail-folded.ll
index beba6a437d880..9ed9d2677fd89 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions-tail-folded.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-reductions-tail-folded.ll
@@ -23,12 +23,12 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (fadd) fast vp<[[VP4]]>, ir<%red.next>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VP9]]>, vp<[[VP8]]>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[VP9]]>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VP10]]>, vp<[[VP8]]>
; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>, vp<[[VP8]]>)
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -37,11 +37,11 @@ define float @print_reduction(i64 %n, ptr noalias %y) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (fadd, in-loop) fast ir<%red.next>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = compute-reduction-result (fadd, in-loop) fast ir<%red.next>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %red.next.lcssa = phi float [ %red.next, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
+; CHECK-NEXT: IR %red.next.lcssa = phi float [ %red.next, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -83,12 +83,12 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (fadd) fast vp<[[VP4]]>, ir<%red.next>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
-; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VP9]]>, vp<[[VP8]]>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[VP9]]>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: WIDEN ir<%lv> = load vp<[[VP10]]>, vp<[[VP8]]>
; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>, vp<[[VP8]]>)
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -97,8 +97,8 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (fadd, in-loop) fast ir<%red.next>
-; CHECK-NEXT: CLONE store vp<[[VP11]]>, ir<%dst>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = compute-reduction-result (fadd, in-loop) fast ir<%red.next>
+; CHECK-NEXT: CLONE store vp<[[VP12]]>, ir<%dst>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
@@ -144,17 +144,17 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%sum.07> = phi (fmuladd) nnan ninf nsz vp<[[VP4]]>, ir<%muladd>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
-; CHECK-NEXT: WIDEN ir<%l.a> = load vp<[[VP9]]>, vp<[[VP8]]>
-; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%b>, vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx2>, ir<1>
-; CHECK-NEXT: WIDEN ir<%l.b> = load vp<[[VP10]]>, vp<[[VP8]]>
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = fmul nnan ninf nsz ir<%l.a>, ir<%l.b>
-; CHECK-NEXT: REDUCE ir<%muladd> = ir<%sum.07> + nnan ninf nsz reduce.fmuladd (vp<[[VP11]]>, vp<[[VP8]]>)
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[VP9]]>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: WIDEN ir<%l.a> = load vp<[[VP10]]>, vp<[[VP8]]>
+; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%b>, vp<[[VP9]]>
+; CHECK-NEXT: vp<[[VP11:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx2>, ir<1>
+; CHECK-NEXT: WIDEN ir<%l.b> = load vp<[[VP11]]>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = fmul nnan ninf nsz ir<%l.a>, ir<%l.b>
+; CHECK-NEXT: REDUCE ir<%muladd> = ir<%sum.07> + nnan ninf nsz reduce.fmuladd (vp<[[VP12]]>, vp<[[VP8]]>)
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -162,11 +162,11 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (fmuladd, in-loop) nnan ninf nsz ir<%muladd>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = compute-reduction-result (fmuladd, in-loop) nnan ninf nsz ir<%muladd>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %muladd.lcssa = phi float [ %muladd, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: IR %muladd.lcssa = phi float [ %muladd, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -210,16 +210,16 @@ define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (find-iv) ir<-9223372036854775808>, vp<[[VP9:%[0-9]+]]>
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (find-iv) ir<-9223372036854775808>, vp<[[VP10:%[0-9]+]]>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP3]]>
-; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr inbounds ir<%a>, vp<[[VP5]]>
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds ir<%gep.a>, ir<1>
-; CHECK-NEXT: WIDEN ir<%l.a> = load vp<[[VP8]]>, vp<[[VP7]]>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr inbounds ir<%a>, vp<[[VP8]]>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds ir<%gep.a>, ir<1>
+; CHECK-NEXT: WIDEN ir<%l.a> = load vp<[[VP9]]>, vp<[[VP7]]>
; CHECK-NEXT: WIDEN ir<%cmp2> = icmp eq ir<%l.a>, ir<%start>
; CHECK-NEXT: WIDEN ir<%cond> = select ir<%cmp2>, ir<%iv>, ir<%rdx>
-; CHECK-NEXT: EMIT vp<[[VP9]]> = select vp<[[VP7]]>, ir<%cond>, ir<%rdx>
+; CHECK-NEXT: EMIT vp<[[VP10]]> = select vp<[[VP7]]>, ir<%cond>, ir<%rdx>
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP4]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -227,13 +227,13 @@ define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (smax) vp<[[VP9]]>
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = icmp ne vp<[[VP11]]>, ir<-9223372036854775808>
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = select vp<[[VP12]]>, vp<[[VP11]]>, ir<%start>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = compute-reduction-result (smax) vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = icmp ne vp<[[VP12]]>, ir<-9223372036854775808>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = select vp<[[VP13]]>, vp<[[VP12]]>, ir<%start>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %cond.lcssa = phi i64 [ %cond, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: IR %cond.lcssa = phi i64 [ %cond, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -275,14 +275,14 @@ define i64 @print_extended_reduction(ptr nocapture readonly %x, ptr nocapture re
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP4]]>, vp<[[VP10:%[0-9]+]]>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP4]]>, vp<[[VP11:%[0-9]+]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
-; CHECK-NEXT: WIDEN ir<%load0> = load vp<[[VP9]]>, vp<[[VP8]]>
-; CHECK-NEXT: EXPRESSION vp<[[VP10]]> = vp<[[VP8]]> + reduce.add (ir<%load0> zext to i64, vp<[[VP8]]>)
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP9]]>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: WIDEN ir<%load0> = load vp<[[VP10]]>, vp<[[VP8]]>
+; CHECK-NEXT: EXPRESSION vp<[[VP11]]> = vp<[[VP8]]> + reduce.add (ir<%load0> zext to i64, vp<[[VP8]]>)
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -290,11 +290,11 @@ define i64 @print_extended_reduction(ptr nocapture readonly %x, ptr nocapture re
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP11]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
+; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -337,17 +337,17 @@ define i64 @print_mulacc(ptr nocapture readonly %x, ptr nocapture readonly %y, i
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP4]]>, vp<[[VP11:%[0-9]+]]>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP4]]>, vp<[[VP12:%[0-9]+]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
-; CHECK-NEXT: WIDEN ir<%load0> = load vp<[[VP9]]>, vp<[[VP8]]>
-; CHECK-NEXT: CLONE ir<%arrayidx1> = getelementptr inbounds ir<%y>, vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx1>, ir<1>
-; CHECK-NEXT: WIDEN ir<%load1> = load vp<[[VP10]]>, vp<[[VP8]]>
-; CHECK-NEXT: EXPRESSION vp<[[VP11]]> = ir<%rdx> + reduce.add (mul nsw ir<%load0>, ir<%load1>, vp<[[VP8]]>)
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP9]]>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: WIDEN ir<%load0> = load vp<[[VP10]]>, vp<[[VP8]]>
+; CHECK-NEXT: CLONE ir<%arrayidx1> = getelementptr inbounds ir<%y>, vp<[[VP9]]>
+; CHECK-NEXT: vp<[[VP11:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx1>, ir<1>
+; CHECK-NEXT: WIDEN ir<%load1> = load vp<[[VP11]]>, vp<[[VP8]]>
+; CHECK-NEXT: EXPRESSION vp<[[VP12]]> = ir<%rdx> + reduce.add (mul nsw ir<%load0>, ir<%load1>, vp<[[VP8]]>)
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -355,11 +355,11 @@ define i64 @print_mulacc(ptr nocapture readonly %x, ptr nocapture readonly %y, i
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP12]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -404,17 +404,17 @@ define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture reado
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP4]]>, vp<[[VP11:%[0-9]+]]>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (add) vp<[[VP4]]>, vp<[[VP12:%[0-9]+]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
-; CHECK-NEXT: WIDEN ir<%load0> = load vp<[[VP9]]>, vp<[[VP8]]>
-; CHECK-NEXT: CLONE ir<%arrayidx1> = getelementptr inbounds ir<%y>, vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx1>, ir<1>
-; CHECK-NEXT: WIDEN ir<%load1> = load vp<[[VP10]]>, vp<[[VP8]]>
-; CHECK-NEXT: EXPRESSION vp<[[VP11]]> = ir<%rdx> + reduce.add (mul nsw (ir<%load0> sext to i64), (ir<%load1> sext to i64), vp<[[VP8]]>)
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP9]]>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: WIDEN ir<%load0> = load vp<[[VP10]]>, vp<[[VP8]]>
+; CHECK-NEXT: CLONE ir<%arrayidx1> = getelementptr inbounds ir<%y>, vp<[[VP9]]>
+; CHECK-NEXT: vp<[[VP11:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx1>, ir<1>
+; CHECK-NEXT: WIDEN ir<%load1> = load vp<[[VP11]]>, vp<[[VP8]]>
+; CHECK-NEXT: EXPRESSION vp<[[VP12]]> = ir<%rdx> + reduce.add (mul nsw (ir<%load0> sext to i64), (ir<%load1> sext to i64), vp<[[VP8]]>)
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -422,11 +422,11 @@ define i64 @print_mulacc_extended(ptr nocapture readonly %x, ptr nocapture reado
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP12]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -474,14 +474,14 @@ define i64 @print_extended_sub_reduction(ptr nocapture readonly %x, ptr nocaptur
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (sub) vp<[[VP4]]>, vp<[[VP10:%[0-9]+]]>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (sub) vp<[[VP4]]>, vp<[[VP11:%[0-9]+]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
-; CHECK-NEXT: WIDEN ir<%load0> = load vp<[[VP9]]>, vp<[[VP8]]>
-; CHECK-NEXT: EXPRESSION vp<[[VP10]]> = vp<[[VP8]]> + reduce.sub (ir<%load0> zext to i64, vp<[[VP8]]>)
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP9]]>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: WIDEN ir<%load0> = load vp<[[VP10]]>, vp<[[VP8]]>
+; CHECK-NEXT: EXPRESSION vp<[[VP11]]> = vp<[[VP8]]> + reduce.sub (ir<%load0> zext to i64, vp<[[VP8]]>)
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -489,11 +489,11 @@ define i64 @print_extended_sub_reduction(ptr nocapture readonly %x, ptr nocaptur
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = compute-reduction-result (sub, in-loop) vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (sub, in-loop) vp<[[VP11]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
+; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -673,17 +673,17 @@ define i64 @print_mulacc_sub_extended(ptr nocapture readonly %x, ptr nocapture r
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (sub) vp<[[VP4]]>, vp<[[VP11:%[0-9]+]]>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (sub) vp<[[VP4]]>, vp<[[VP12:%[0-9]+]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
-; CHECK-NEXT: WIDEN ir<%load0> = load vp<[[VP9]]>, vp<[[VP8]]>
-; CHECK-NEXT: CLONE ir<%arrayidx1> = getelementptr inbounds ir<%y>, vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx1>, ir<1>
-; CHECK-NEXT: WIDEN ir<%load1> = load vp<[[VP10]]>, vp<[[VP8]]>
-; CHECK-NEXT: EXPRESSION vp<[[VP11]]> = ir<%rdx> + reduce.sub (mul nsw (ir<%load0> sext to i64), (ir<%load1> sext to i64), vp<[[VP8]]>)
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP9]]>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: WIDEN ir<%load0> = load vp<[[VP10]]>, vp<[[VP8]]>
+; CHECK-NEXT: CLONE ir<%arrayidx1> = getelementptr inbounds ir<%y>, vp<[[VP9]]>
+; CHECK-NEXT: vp<[[VP11:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx1>, ir<1>
+; CHECK-NEXT: WIDEN ir<%load1> = load vp<[[VP11]]>, vp<[[VP8]]>
+; CHECK-NEXT: EXPRESSION vp<[[VP12]]> = ir<%rdx> + reduce.sub (mul nsw (ir<%load0> sext to i64), (ir<%load1> sext to i64), vp<[[VP8]]>)
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -691,11 +691,11 @@ define i64 @print_mulacc_sub_extended(ptr nocapture readonly %x, ptr nocapture r
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (sub, in-loop) vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = compute-reduction-result (sub, in-loop) vp<[[VP12]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -743,14 +743,14 @@ define i64 @print_mulacc_duplicate_extends(ptr nocapture readonly %x, ptr nocapt
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (sub) vp<[[VP4]]>, vp<[[VP10:%[0-9]+]]>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi (sub) vp<[[VP4]]>, vp<[[VP11:%[0-9]+]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
-; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP6]]>
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
-; CHECK-NEXT: WIDEN ir<%load0> = load vp<[[VP9]]>, vp<[[VP8]]>
-; CHECK-NEXT: EXPRESSION vp<[[VP10]]> = ir<%rdx> + reduce.sub (mul nsw (ir<%load0> sext to i64), (ir<%load0> sext to i64), vp<[[VP8]]>)
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%x>, vp<[[VP9]]>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer inbounds ir<%arrayidx>, ir<1>
+; CHECK-NEXT: WIDEN ir<%load0> = load vp<[[VP10]]>, vp<[[VP8]]>
+; CHECK-NEXT: EXPRESSION vp<[[VP11]]> = ir<%rdx> + reduce.sub (mul nsw (ir<%load0> sext to i64), (ir<%load0> sext to i64), vp<[[VP8]]>)
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -758,11 +758,11 @@ define i64 @print_mulacc_duplicate_extends(ptr nocapture readonly %x, ptr nocapt
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = compute-reduction-result (sub, in-loop) vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (sub, in-loop) vp<[[VP11]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP12]]> from middle.block)
+; CHECK-NEXT: IR %r.0.lcssa = phi i64 [ %rdx.next, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -808,14 +808,14 @@ define i32 @print_mulacc_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP5]]>, vp<[[VP11:%[0-9]+]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP7]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP5]]>, vp<[[VP12:%[0-9]+]]>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp ule vp<[[VP8]]>, vp<[[VP3]]>
-; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
-; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VP10]]>, vp<[[VP9]]>
-; CHECK-NEXT: EXPRESSION vp<[[VP11]]> = ir<%red> + reduce.add (mul (ir<%l> zext to i32), (ir<63> zext to i32), vp<[[VP9]]>)
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP10]]>
+; CHECK-NEXT: vp<[[VP11:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
+; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VP11]]>, vp<[[VP9]]>
+; CHECK-NEXT: EXPRESSION vp<[[VP12]]> = ir<%red> + reduce.add (mul (ir<%l> zext to i32), (ir<63> zext to i32), vp<[[VP9]]>)
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP6]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -823,11 +823,11 @@ define i32 @print_mulacc_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP12]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %red.next.lcssa = phi i32 [ %red.next, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: IR %red.next.lcssa = phi i32 [ %red.next, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -870,15 +870,15 @@ define i32 @print_mulacc_extended_const_lhs(ptr %start, ptr %end) {
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP5]]>, vp<[[VP11:%[0-9]+]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP7]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP5]]>, vp<[[VP12:%[0-9]+]]>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp ule vp<[[VP8]]>, vp<[[VP3]]>
-; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
-; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VP10]]>, vp<[[VP9]]>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP10]]>
+; CHECK-NEXT: vp<[[VP11:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
+; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VP11]]>, vp<[[VP9]]>
; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = zext ir<%l> to i32
-; CHECK-NEXT: EXPRESSION vp<[[VP11]]> = ir<%red> + reduce.add (mul ir<63>, ir<%l.ext>, vp<[[VP9]]>)
+; CHECK-NEXT: EXPRESSION vp<[[VP12]]> = ir<%red> + reduce.add (mul ir<63>, ir<%l.ext>, vp<[[VP9]]>)
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP6]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -886,11 +886,11 @@ define i32 @print_mulacc_extended_const_lhs(ptr %start, ptr %end) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP12]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %red.next.lcssa = phi i32 [ %red.next, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: IR %red.next.lcssa = phi i32 [ %red.next, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -934,15 +934,15 @@ define i32 @print_mulacc_not_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP5]]>, vp<[[VP11:%[0-9]+]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP7]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP5]]>, vp<[[VP12:%[0-9]+]]>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp ule vp<[[VP8]]>, vp<[[VP3]]>
-; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
-; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VP10]]>, vp<[[VP9]]>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP10]]>
+; CHECK-NEXT: vp<[[VP11:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
+; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VP11]]>, vp<[[VP9]]>
; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = sext ir<%l> to i32
-; CHECK-NEXT: EXPRESSION vp<[[VP11]]> = ir<%red> + reduce.add (mul ir<%l.ext>, ir<128>, vp<[[VP9]]>)
+; CHECK-NEXT: EXPRESSION vp<[[VP12]]> = ir<%red> + reduce.add (mul ir<%l.ext>, ir<128>, vp<[[VP9]]>)
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP6]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -950,11 +950,11 @@ define i32 @print_mulacc_not_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP12]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %red.next.lcssa = phi i32 [ %red.next, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: IR %red.next.lcssa = phi i32 [ %red.next, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -998,14 +998,14 @@ define i64 @print_ext_mulacc_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP5]]>, vp<[[VP11:%[0-9]+]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP7]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP5]]>, vp<[[VP12:%[0-9]+]]>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp ule vp<[[VP8]]>, vp<[[VP3]]>
-; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
-; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VP10]]>, vp<[[VP9]]>
-; CHECK-NEXT: EXPRESSION vp<[[VP11]]> = ir<%red> + reduce.add (mul (ir<%l> zext to i64), (ir<63> zext to i64), vp<[[VP9]]>)
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP10]]>
+; CHECK-NEXT: vp<[[VP11:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
+; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VP11]]>, vp<[[VP9]]>
+; CHECK-NEXT: EXPRESSION vp<[[VP12]]> = ir<%red> + reduce.add (mul (ir<%l> zext to i64), (ir<63> zext to i64), vp<[[VP9]]>)
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP6]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -1013,11 +1013,11 @@ define i64 @print_ext_mulacc_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP12]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %red.next.lcssa = phi i64 [ %red.next, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: IR %red.next.lcssa = phi i64 [ %red.next, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -1062,16 +1062,16 @@ define i64 @print_ext_mulacc_not_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP5]]>, vp<[[VP12:%[0-9]+]]>
-; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP7]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi (add) vp<[[VP5]]>, vp<[[VP13:%[0-9]+]]>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP6]]>
; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = icmp ule vp<[[VP8]]>, vp<[[VP3]]>
-; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
-; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VP10]]>, vp<[[VP9]]>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<%next.gep> = ptradd ir<%start>, vp<[[VP10]]>
+; CHECK-NEXT: vp<[[VP11:%[0-9]+]]> = vector-pointer vp<%next.gep>, ir<1>
+; CHECK-NEXT: WIDEN ir<%l> = load vp<[[VP11]]>, vp<[[VP9]]>
; CHECK-NEXT: WIDEN-CAST ir<%l.ext> = sext ir<%l> to i32
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = shl ir<%l.ext>, ir<7>
-; CHECK-NEXT: EXPRESSION vp<[[VP12]]> = vp<[[VP9]]> + reduce.add (vp<[[VP11]]> sext to i64, vp<[[VP9]]>)
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = shl ir<%l.ext>, ir<7>
+; CHECK-NEXT: EXPRESSION vp<[[VP13]]> = vp<[[VP9]]> + reduce.add (vp<[[VP12]]> sext to i64, vp<[[VP9]]>)
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP6]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -1079,11 +1079,11 @@ define i64 @print_ext_mulacc_not_extended_const(ptr %start, ptr %end) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP12]]>
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP13]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %red.next.lcssa = phi i64 [ %red.next, %loop ] (extra operand: vp<[[VP14]]> from middle.block)
+; CHECK-NEXT: IR %red.next.lcssa = phi i64 [ %red.next, %loop ] (extra operand: vp<[[VP15]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
@@ -1131,10 +1131,10 @@ define i64 @print_ext_mul_two_uses(i64 %n, ptr %a, i16 %b, i32 %c) {
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%res2> = phi (add) vp<[[VP4]]>, vp<[[VP8:%[0-9]+]]>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP2]]>
-; CHECK-NEXT: EXPRESSION vp<[[VP8]]> = ir<%res2> + reduce.add (mul (ir<%b> sext to i64), (ir<%b> sext to i64), vp<[[VP7]]>)
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%res2> = phi (add) vp<[[VP4]]>, vp<[[VP9:%[0-9]+]]>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP2]]>
+; CHECK-NEXT: EXPRESSION vp<[[VP9]]> = ir<%res2> + reduce.add (mul (ir<%b> sext to i64), (ir<%b> sext to i64), vp<[[VP8]]>)
; CHECK-NEXT: EMIT vp<%index.next> = add vp<[[VP5]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP1]]>
; CHECK-NEXT: No successors
@@ -1142,11 +1142,11 @@ define i64 @print_ext_mul_two_uses(i64 %n, ptr %a, i16 %b, i32 %c) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = compute-reduction-result (add, in-loop) vp<[[VP9]]>
; CHECK-NEXT: Successor(s): ir-bb<exit>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: IR %add.lcssa = phi i64 [ %add, %loop ] (extra operand: vp<[[VP10]]> from middle.block)
+; CHECK-NEXT: IR %add.lcssa = phi i64 [ %add, %loop ] (extra operand: vp<[[VP11]]> from middle.block)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
index c7c3d7cfa6617..2e9c7e7880795 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-sink-scalars-and-merge.ll
@@ -28,42 +28,42 @@ define void @sink1(i32 %k, i32 %x) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: WIDEN ir<%cond> = icmp eq ir<%iv>, ir<%x>
-; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP8]]>, ir<%cond>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = logical-and vp<[[VP8]]>, ir<%cond>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
; CHECK-NEXT: pred.load.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP10]]>
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[VP6]]>
+; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[VP9]]>
; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b>
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP10:%[0-9]+]]> = ir<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP11:%[0-9]+]]> = ir<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): if.0
; CHECK-EMPTY:
; CHECK-NEXT: if.0:
-; CHECK-NEXT: REPLICATE ir<%add> = add vp<[[VP10]]>, ir<10>
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = shl ir<%add>, ir<1>
+; CHECK-NEXT: REPLICATE ir<%add> = add vp<[[VP11]]>, ir<10>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = shl ir<%add>, ir<1>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP10]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[VP6]]>
-; CHECK-NEXT: REPLICATE store vp<[[VP11]]>, ir<%gep.a>
+; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[VP9]]>
+; CHECK-NEXT: REPLICATE store vp<[[VP12]]>, ir<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
@@ -141,39 +141,39 @@ define void @sink2(i32 %k) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP3]]>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
; CHECK-NEXT: pred.load.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP7]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP8]]>
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[VP8]]>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[VP9]]>
; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b>
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP9:%[0-9]+]]> = ir<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP10:%[0-9]+]]> = ir<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:
; CHECK-NEXT: loop.0:
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = shl ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = shl ir<%iv>, ir<1>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP7]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP8]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[VP10]]>
-; CHECK-NEXT: REPLICATE ir<%add> = add vp<[[VP9]]>, ir<10>
+; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[VP11]]>
+; CHECK-NEXT: REPLICATE ir<%add> = add vp<[[VP10]]>, ir<10>
; CHECK-NEXT: REPLICATE store ir<%add>, ir<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
@@ -254,35 +254,35 @@ define void @sink3(i32 %k) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP3]]>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
; CHECK-NEXT: pred.load.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP7]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP8]]>
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[VP8]]>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[VP9]]>
; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V)
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP9:%[0-9]+]]> = ir<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP10:%[0-9]+]]> = ir<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:
; CHECK-NEXT: loop.0:
-; CHECK-NEXT: WIDEN ir<%add> = add vp<[[VP9]]>, ir<10>
+; CHECK-NEXT: WIDEN ir<%add> = add vp<[[VP10]]>, ir<10>
; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<%add>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP7]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP8]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
@@ -367,22 +367,22 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<21>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = DERIVED-IV ir<21> + vp<[[VP4]]> * ir<1>
; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP4]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP3]]>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = DERIVED-IV ir<21> + vp<[[VP4]]> * ir<1>
; CHECK-NEXT: CLONE ir<%lv> = load ir<%A>
; CHECK-NEXT: WIDEN ir<%cmp> = icmp uge ir<%iv>, ir<%k>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP7]]>, ir<%cmp>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP7]]>, ir<%cmp>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP8]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: REPLICATE ir<%gep.B> = getelementptr inbounds ir<%B>, vp<[[VP9]]>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = SCALAR-STEPS vp<[[VP8]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: REPLICATE ir<%gep.B> = getelementptr inbounds ir<%B>, vp<[[VP10]]>
; CHECK-NEXT: REPLICATE store ir<%lv>, ir<%gep.B>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
@@ -460,37 +460,37 @@ define void @pred_cfg1(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP3]]>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
; CHECK-NEXT: WIDEN ir<%c.1> = icmp ult ir<%iv>, ir<%j>
; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP7]]>, ir<%c.1>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP8]]>, ir<%c.1>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
; CHECK-NEXT: pred.load.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP8]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]>
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[VP9]]>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[VP10]]>
; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V)
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP10:%[0-9]+]]> = ir<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP11:%[0-9]+]]> = ir<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): then.0.0
; CHECK-EMPTY:
; CHECK-NEXT: then.0.0:
-; CHECK-NEXT: BLEND ir<%p> = ir<0> vp<%10>/ir<%c.1>
+; CHECK-NEXT: BLEND ir<%p> = ir<0> vp<%11>/ir<%c.1>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP7]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP8]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
@@ -578,39 +578,39 @@ define void @pred_cfg2(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP3]]>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10>
; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j>
; CHECK-NEXT: WIDEN ir<%c.1> = icmp ugt ir<%iv>, ir<%j>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP7]]>, ir<%c.0>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP8]]>, ir<%c.0>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
; CHECK-NEXT: pred.load.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP8]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]>
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[VP9]]>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[VP10]]>
; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V)
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP10:%[0-9]+]]> = ir<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP11:%[0-9]+]]> = ir<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): then.0.0
; CHECK-EMPTY:
; CHECK-NEXT: then.0.0:
-; CHECK-NEXT: BLEND ir<%p> = ir<0> vp<%10>/ir<%c.0>
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = logical-and vp<[[VP7]]>, ir<%c.1>
+; CHECK-NEXT: BLEND ir<%p> = ir<0> vp<%11>/ir<%c.0>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP8]]>, ir<%c.1>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP11]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP12]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
@@ -706,38 +706,38 @@ define void @pred_cfg3(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP3]]>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%iv>, ir<10>
; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = logical-and vp<[[VP7]]>, ir<%c.0>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = logical-and vp<[[VP8]]>, ir<%c.0>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
; CHECK-NEXT: pred.load.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP8]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP9]]>
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
-; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[VP9]]>
+; CHECK-NEXT: vp<[[VP10:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[VP10]]>
; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V)
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP10:%[0-9]+]]> = ir<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP11:%[0-9]+]]> = ir<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): then.0.0
; CHECK-EMPTY:
; CHECK-NEXT: then.0.0:
-; CHECK-NEXT: BLEND ir<%p> = ir<0> vp<%10>/ir<%c.0>
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = logical-and vp<[[VP7]]>, ir<%c.0>
+; CHECK-NEXT: BLEND ir<%p> = ir<0> vp<%11>/ir<%c.0>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP8]]>, ir<%c.0>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP11]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP12]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
@@ -834,9 +834,9 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) {
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -845,35 +845,35 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[VP6]]>
+; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[VP9]]>
; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a> (S->V)
-; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[VP6]]>
+; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr inbounds ir<@b>, ir<0>, vp<[[VP9]]>
; CHECK-NEXT: REPLICATE ir<%lv.b> = load ir<%gep.b> (S->V)
-; CHECK-NEXT: REPLICATE ir<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, vp<[[VP6]]>
+; CHECK-NEXT: REPLICATE ir<%gep.c> = getelementptr inbounds ir<@c>, ir<0>, vp<[[VP9]]>
; CHECK-NEXT: REPLICATE store ir<%lv.a>, ir<%gep.c>
; CHECK-NEXT: REPLICATE store ir<%lv.b>, ir<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP9:%[0-9]+]]> = ir<%lv.a>
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP10:%[0-9]+]]> = ir<%lv.b>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP10:%[0-9]+]]> = ir<%lv.a>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP11:%[0-9]+]]> = ir<%lv.b>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.3
; CHECK-EMPTY:
; CHECK-NEXT: loop.3:
; CHECK-NEXT: WIDEN ir<%c.0> = icmp ult ir<%iv>, ir<%j>
-; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = logical-and vp<[[VP8]]>, ir<%c.0>
-; CHECK-NEXT: WIDEN ir<%mul> = mul vp<[[VP9]]>, vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = logical-and vp<[[VP8]]>, ir<%c.0>
+; CHECK-NEXT: WIDEN ir<%mul> = mul vp<[[VP10]]>, vp<[[VP11]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP11]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP12]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%gep.c.1> = getelementptr inbounds ir<@c>, ir<0>, vp<[[VP6]]>
+; CHECK-NEXT: REPLICATE ir<%gep.c.1> = getelementptr inbounds ir<@c>, ir<0>, vp<[[VP9]]>
; CHECK-NEXT: REPLICATE store ir<%mul>, ir<%gep.c.1>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
@@ -963,18 +963,18 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) {
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = icmp ule vp<[[VP6]]>, vp<[[VP3]]>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
; CHECK-NEXT: pred.store.entry:
-; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP7]]>
+; CHECK-NEXT: BRANCH-ON-MASK vp<[[VP8]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[VP8]]>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[VP9]]>
; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a>
; CHECK-NEXT: REPLICATE ir<%div> = sdiv ir<%lv.a>, ir<%lv.a>
; CHECK-NEXT: REPLICATE store ir<%div>, ir<%gep.a>
@@ -1052,11 +1052,11 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<[[VP9:%[0-9]+]]>
-; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<%for> = phi ir<0>, vp<[[VP10:%[0-9]+]]>
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION nuw vp<[[VP5]]>
; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = icmp ule vp<[[VP7]]>, vp<[[VP3]]>
-; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[VP6]]>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr inbounds ir<@a>, ir<0>, vp<[[VP9]]>
; CHECK-NEXT: Successor(s): pred.load
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.load: {
@@ -1069,13 +1069,13 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
; CHECK-NEXT: Successor(s): pred.load.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.continue:
-; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP9]]> = ir<%lv.a>
+; CHECK-NEXT: PHI-PREDICATED-INSTRUCTION vp<[[VP10]]> = ir<%lv.a>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:
; CHECK-NEXT: loop.0:
-; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = first-order splice ir<%for>, vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = first-order splice ir<%for>, vp<[[VP10]]>
; CHECK-NEXT: Successor(s): pred.store
; CHECK-EMPTY:
; CHECK-NEXT: <xVFxUF> pred.store: {
@@ -1084,7 +1084,7 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
-; CHECK-NEXT: REPLICATE ir<%div> = sdiv vp<[[VP10]]>, vp<[[VP9]]>
+; CHECK-NEXT: REPLICATE ir<%div> = sdiv vp<[[VP11]]>, vp<[[VP10]]>
; CHECK-NEXT: REPLICATE store ir<%div>, ir<%gep.a>
; CHECK-NEXT: Successor(s): pred.store.continue
; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index 8313a8a726b16..ea98c4c116952 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -1040,8 +1040,8 @@ define void @known_deref_load_tail_folding() #4 {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 1, [[INDEX]]
; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 10)
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 1, [[INDEX]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr double, ptr @src.arr, i64 [[TMP0]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 8 [[TMP2]], <4 x i1> [[TMP1]], <4 x double> poison)
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/X86/drop-inbounds-flags-for-reverse-vector-pointer.ll b/llvm/test/Transforms/LoopVectorize/X86/drop-inbounds-flags-for-reverse-vector-pointer.ll
index e16fa59db06b4..8c63de93f4325 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/drop-inbounds-flags-for-reverse-vector-pointer.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/drop-inbounds-flags-for-reverse-vector-pointer.ll
@@ -16,8 +16,8 @@ define i1 @fn(ptr %nno) #0 {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 10, i64 9, i64 8, i64 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND1:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 10, [[INDEX]]
; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <4 x i8> [[VEC_IND1]], splat (i8 10)
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 10, [[INDEX]]
; CHECK-NEXT: [[TMP2:%.*]] = and <4 x i64> [[VEC_IND]], splat (i64 1)
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i64> [[TMP2]], zeroinitializer
; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds nuw i32, ptr [[NNO]], i64 [[OFFSET_IDX]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/fold-tail-low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/X86/fold-tail-low-trip-count.ll
index 561008b37964a..65729c941696b 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/fold-tail-low-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/fold-tail-low-trip-count.ll
@@ -61,8 +61,8 @@ define ptr @low_trip_count_via_profile_info_with_iv_live_out(ptr align 16 %start
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IV:%.*]] = phi <64 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31, i64 32, i64 33, i64 34, i64 35, i64 36, i64 37, i64 38, i64 39, i64 40, i64 41, i64 42, i64 43, i64 44, i64 45, i64 46, i64 47, i64 48, i64 49, i64 50, i64 51, i64 52, i64 53, i64 54, i64 55, i64 56, i64 57, i64 58, i64 59, i64 60, i64 61, i64 62, i64 63>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ule <64 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP4]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0(ptr align 1 [[TMP5]], <64 x i1> [[TMP3]], <64 x i8> poison)
@@ -114,8 +114,8 @@ define void @low_trip_count_via_profile_info(ptr align 16 %start, ptr align 16 %
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IV:%.*]] = phi <64 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15, i64 16, i64 17, i64 18, i64 19, i64 20, i64 21, i64 22, i64 23, i64 24, i64 25, i64 26, i64 27, i64 28, i64 29, i64 30, i64 31, i64 32, i64 33, i64 34, i64 35, i64 36, i64 37, i64 38, i64 39, i64 40, i64 41, i64 42, i64 43, i64 44, i64 45, i64 46, i64 47, i64 48, i64 49, i64 50, i64 51, i64 52, i64 53, i64 54, i64 55, i64 56, i64 57, i64 58, i64 59, i64 60, i64 61, i64 62, i64 63>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <64 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP3]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <64 x i8> @llvm.masked.load.v64i8.p0(ptr align 1 [[TMP4]], <64 x i1> [[TMP2]], <64 x i8> poison)
diff --git a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
index 72ba0845e244a..e6420f8735b03 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
@@ -584,8 +584,8 @@ define void @wide_iv_trunc(ptr %dst, i64 %N) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[INDEX]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[INDEX]] to i32
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
; CHECK-NEXT: br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr81872.ll b/llvm/test/Transforms/LoopVectorize/X86/pr81872.ll
index 68b4f6aed25f1..24d5e36f6b67a 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr81872.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr81872.ll
@@ -21,8 +21,8 @@ define void @test(ptr noundef align 8 dereferenceable_or_null(16) %arr) #0 {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 99, i64 98, i64 97, i64 96>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND1:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 99, [[INDEX]]
; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i8> [[VEC_IND1]], splat (i8 14)
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 99, [[INDEX]]
; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i64> [[VEC_IND]], splat (i64 1)
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[TMP1]], zeroinitializer
; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP0]], <4 x i1> [[TMP2]], <4 x i1> zeroinitializer
diff --git a/llvm/test/Transforms/LoopVectorize/X86/small-size.ll b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
index eccbf8b845e73..9040c7381b3ba 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/small-size.ll
@@ -141,8 +141,8 @@ define void @example2(i32 %n, i32 %x) optsize {
; CHECK: vector.body14:
; CHECK-NEXT: [[INDEX16:%.*]] = phi i64 [ 0, [[VECTOR_PH8]] ], [ [[INDEX_NEXT29:%.*]], [[PRED_STORE_CONTINUE26:%.*]] ]
; CHECK-NEXT: [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH8]] ], [ [[VEC_IND_NEXT26:%.*]], [[PRED_STORE_CONTINUE26]] ]
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[I_0_LCSSA]], [[INDEX16]]
; CHECK-NEXT: [[TMP18:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT20]]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[I_0_LCSSA]], [[INDEX16]]
; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP18]], i64 0
; CHECK-NEXT: br i1 [[TMP19]], label [[PRED_STORE_IF19:%.*]], label [[PRED_STORE_CONTINUE20:%.*]]
; CHECK: pred.store.if17:
@@ -267,6 +267,7 @@ define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE16:%.*]] ]
; CHECK-NEXT: [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE16]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT12]]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[OFFSET_IDX]], 4
; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[OFFSET_IDX]], 8
@@ -279,7 +280,6 @@ define void @example3(i32 %n, ptr noalias nocapture %p, ptr noalias nocapture %q
; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP12]]
; CHECK-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP15]]
; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP3:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT12]]
; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0
; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; CHECK: pred.store.if:
@@ -434,6 +434,7 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst)
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE15:%.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE15]] ]
+; CHECK-NEXT: [[TMP27:%.*]] = icmp ule <4 x i16> [[VEC_IND]], splat (i16 256)
; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP1]], 2
; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[TMP1]], 4
@@ -450,7 +451,6 @@ define void @example23c(ptr noalias nocapture %src, ptr noalias nocapture %dst)
; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP28]]
; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP29]]
; CHECK-NEXT: [[NEXT_GEP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP8]]
-; CHECK-NEXT: [[TMP27:%.*]] = icmp ule <4 x i16> [[VEC_IND]], splat (i16 256)
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP27]], i64 0
; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; CHECK: pred.store.if:
@@ -526,6 +526,7 @@ define i64 @example23d(ptr noalias nocapture %src, ptr noalias nocapture %dst) o
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE14:%.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE14]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ule <4 x i16> [[VEC_IND]], splat (i16 256)
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], 2
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 4
@@ -542,7 +543,6 @@ define i64 @example23d(ptr noalias nocapture %src, ptr noalias nocapture %dst) o
; CHECK-NEXT: [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP25]]
; CHECK-NEXT: [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP26]]
; CHECK-NEXT: [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ule <4 x i16> [[VEC_IND]], splat (i16 256)
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP7]], i64 0
; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; CHECK: pred.store.if:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
index c350f05d0cbe1..ead2f8481e539 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
@@ -61,9 +61,9 @@ define void @test_pr59090_interleave(ptr %l_out, ptr noalias %b) #0 {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <8 x i64> [[VEC_IND]], splat (i64 8)
-; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <8 x i64> [[VEC_IND]], splat (i64 10000)
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <8 x i64> [[STEP_ADD]], splat (i64 10000)
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 8
; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[INDEX]], 6
; CHECK-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP0]], 6
; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[B:%.*]], align 1, !llvm.access.group [[ACC_GRP0]]
diff --git a/llvm/test/Transforms/LoopVectorize/alias-mask.ll b/llvm/test/Transforms/LoopVectorize/alias-mask.ll
index be314fbafff04..841035c96c7a9 100644
--- a/llvm/test/Transforms/LoopVectorize/alias-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/alias-mask.ll
@@ -606,8 +606,8 @@ define i64 @any_of_reduction(i64 %a, ptr %src, ptr %dst, i32 %n) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = add i32 1, [[INDEX]]
; CHECK-NEXT: [[HEADER_MASK:%.*]] = icmp ule <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP10:%.*]] = add i32 1, [[INDEX]]
; CHECK-NEXT: [[CLAMPED_HEADER_MASK:%.*]] = and <4 x i1> [[HEADER_MASK]], [[ALIAS_MASK]]
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[TMP10]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP13]], <4 x i1> [[CLAMPED_HEADER_MASK]], <4 x i32> poison)
diff --git a/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll
index 2aa7dfd5ee8ea..264ae9d1b3471 100644
--- a/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/find-last-iv-sinkable-expr-tail-folding.ll
@@ -192,9 +192,9 @@ define i64 @findlast_non_canonical_iv_with_expr(ptr %a, i64 %n) {
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 10, i64 12, i64 14, i64 16>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE8]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP36:%.*]], %[[PRED_LOAD_CONTINUE8]] ]
; CHECK-NEXT: [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT8:%.*]], %[[PRED_LOAD_CONTINUE8]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[INDEX]], 1
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 10, [[TMP3]]
-; CHECK-NEXT: [[TMP4:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i64 0
; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
; CHECK: [[PRED_LOAD_IF]]:
@@ -454,8 +454,8 @@ define i64 @findlast_sdiv_iv_as_divisor(ptr %a, i64 %n) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP3]], <4 x i1> [[TMP2]], <4 x i64> poison)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
@@ -525,8 +525,8 @@ define i64 @findlast_expr_flipped_select_anyof(ptr %a, ptr %b, i64 %rdx.start, i
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT8:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[N]], [[INDEX]]
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[N]], [[INDEX]]
; CHECK-NEXT: [[TMP3:%.*]] = add nsw i64 [[OFFSET_IDX]], -1
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[TMP4]], i64 -3
@@ -714,8 +714,8 @@ define i64 @findlast_udiv_may_trap_due_to_sentinel(ptr %a, i64 %n) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 -9223372036854775808, [[INDEX]]
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 -9223372036854775808, [[INDEX]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP3]], <4 x i1> [[TMP2]], <4 x i64> poison)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
@@ -776,8 +776,8 @@ define i64 @findlast_srem_iv_as_divisor(ptr %a, i64 %n) {
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 -1), %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 8 [[TMP3]], <4 x i1> [[TMP2]], <4 x i64> poison)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i64> [[WIDE_MASKED_LOAD]], splat (i64 42)
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-tail-folding.ll
index 30b163d1692c6..7373fa2a96407 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-tail-folding.ll
@@ -107,8 +107,8 @@ define i32 @FOR_used_outside(ptr noalias %A, ptr noalias %B, i64 %n) {
; VF1IC2: [[VECTOR_BODY]]:
; VF1IC2-NEXT: [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE5:.*]] ]
; VF1IC2-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ 33, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[PRED_STORE_CONTINUE5]] ]
-; VF1IC2-NEXT: [[VEC_IV1:%.*]] = add i64 [[TMP0]], 1
; VF1IC2-NEXT: [[VEC_IV:%.*]] = add i64 [[TMP0]], 0
+; VF1IC2-NEXT: [[VEC_IV1:%.*]] = add i64 [[TMP0]], 1
; VF1IC2-NEXT: [[TMP1:%.*]] = icmp ule i64 [[VEC_IV]], [[TRIP_COUNT_MINUS_1]]
; VF1IC2-NEXT: [[TMP2:%.*]] = icmp ule i64 [[VEC_IV1]], [[TRIP_COUNT_MINUS_1]]
; VF1IC2-NEXT: br i1 [[TMP1]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
@@ -284,8 +284,8 @@ define i32 @FOR_next_used_outside(ptr noalias %A, ptr noalias %B, i64 %n) {
; VF1IC2: [[VECTOR_BODY]]:
; VF1IC2-NEXT: [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE5:.*]] ]
; VF1IC2-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ 33, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[PRED_STORE_CONTINUE5]] ]
-; VF1IC2-NEXT: [[VEC_IV1:%.*]] = add i64 [[TMP0]], 1
; VF1IC2-NEXT: [[VEC_IV:%.*]] = add i64 [[TMP0]], 0
+; VF1IC2-NEXT: [[VEC_IV1:%.*]] = add i64 [[TMP0]], 1
; VF1IC2-NEXT: [[TMP1:%.*]] = icmp ule i64 [[VEC_IV]], [[TRIP_COUNT_MINUS_1]]
; VF1IC2-NEXT: [[TMP2:%.*]] = icmp ule i64 [[VEC_IV1]], [[TRIP_COUNT_MINUS_1]]
; VF1IC2-NEXT: br i1 [[TMP1]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
@@ -469,8 +469,8 @@ define i32 @FOR_and_next_used_outside(ptr noalias %A, ptr noalias %B, i64 %n) {
; VF1IC2: [[VECTOR_BODY]]:
; VF1IC2-NEXT: [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE5:.*]] ]
; VF1IC2-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ 33, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[PRED_STORE_CONTINUE5]] ]
-; VF1IC2-NEXT: [[VEC_IV1:%.*]] = add i64 [[TMP0]], 1
; VF1IC2-NEXT: [[VEC_IV:%.*]] = add i64 [[TMP0]], 0
+; VF1IC2-NEXT: [[VEC_IV1:%.*]] = add i64 [[TMP0]], 1
; VF1IC2-NEXT: [[TMP1:%.*]] = icmp ule i64 [[VEC_IV]], [[TRIP_COUNT_MINUS_1]]
; VF1IC2-NEXT: [[TMP2:%.*]] = icmp ule i64 [[VEC_IV1]], [[TRIP_COUNT_MINUS_1]]
; VF1IC2-NEXT: br i1 [[TMP1]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
index 100afc7049ebc..241c4bb4de3aa 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
@@ -2617,9 +2617,9 @@ define i32 @sink_into_replication_region(i32 %y) {
; UNROLL-NO-IC-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP47:%.*]], %[[PRED_UDIV_CONTINUE20]] ]
; UNROLL-NO-IC-NEXT: [[VEC_IV:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_UDIV_CONTINUE20]] ]
; UNROLL-NO-IC-NEXT: [[VEC_IV4:%.*]] = add nuw <4 x i32> [[VEC_IV]], splat (i32 4)
-; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]]
; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = icmp ule <4 x i32> [[VEC_IV]], [[BROADCAST_SPLAT6]]
; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = icmp ule <4 x i32> [[VEC_IV4]], [[BROADCAST_SPLAT6]]
+; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]]
; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
; UNROLL-NO-IC-NEXT: br i1 [[TMP4]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
; UNROLL-NO-IC: [[PRED_UDIV_IF]]:
@@ -2726,11 +2726,11 @@ define i32 @sink_into_replication_region(i32 %y) {
; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[PRED_UDIV_CONTINUE4]] ]
; UNROLL-NO-VF-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[PRED_UDIV_CONTINUE4]] ]
; UNROLL-NO-VF-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[PRED_UDIV_CONTINUE4]] ]
-; UNROLL-NO-VF-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]]
; UNROLL-NO-VF-NEXT: [[VEC_IV:%.*]] = add i32 [[INDEX]], 0
; UNROLL-NO-VF-NEXT: [[VEC_IV2:%.*]] = add i32 [[INDEX]], 1
; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = icmp ule i32 [[VEC_IV]], [[TRIP_COUNT_MINUS_1]]
; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = icmp ule i32 [[VEC_IV2]], [[TRIP_COUNT_MINUS_1]]
+; UNROLL-NO-VF-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]]
; UNROLL-NO-VF-NEXT: br i1 [[TMP2]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
; UNROLL-NO-VF: [[PRED_UDIV_IF]]:
; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = udiv i32 219220132, [[OFFSET_IDX]]
@@ -2777,8 +2777,8 @@ define i32 @sink_into_replication_region(i32 %y) {
; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[PRED_UDIV_CONTINUE8]] ]
; SINK-AFTER-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[PRED_UDIV_CONTINUE8]] ]
; SINK-AFTER-NEXT: [[VEC_IV:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_UDIV_CONTINUE8]] ]
-; SINK-AFTER-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]]
; SINK-AFTER-NEXT: [[TMP2:%.*]] = icmp ule <4 x i32> [[VEC_IV]], [[BROADCAST_SPLAT2]]
+; SINK-AFTER-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]]
; SINK-AFTER-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0
; SINK-AFTER-NEXT: br i1 [[TMP3]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
; SINK-AFTER: [[PRED_UDIV_IF]]:
@@ -2868,6 +2868,12 @@ define i32 @sink_into_replication_region_multiple(ptr %x, i32 %y) {
; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, %[[VECTOR_PH]] ], [ [[TMP43:%.*]], %[[PRED_STORE_CONTINUE29]] ]
; UNROLL-NO-IC-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP46:%.*]], %[[PRED_STORE_CONTINUE29]] ]
; UNROLL-NO-IC-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP47:%.*]], %[[PRED_STORE_CONTINUE29]] ]
+; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i32> poison, i32 [[INDEX]], i64 0
+; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT2]], <4 x i32> poison, <4 x i32> zeroinitializer
+; UNROLL-NO-IC-NEXT: [[VEC_IV:%.*]] = add nuw <4 x i32> [[BROADCAST_SPLAT3]], <i32 0, i32 1, i32 2, i32 3>
+; UNROLL-NO-IC-NEXT: [[VEC_IV4:%.*]] = add nuw <4 x i32> [[BROADCAST_SPLAT3]], <i32 4, i32 5, i32 6, i32 7>
+; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = icmp ule <4 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = icmp ule <4 x i32> [[VEC_IV4]], [[BROADCAST_SPLAT]]
; UNROLL-NO-IC-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]]
; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add i32 [[OFFSET_IDX]], -1
; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = add i32 [[OFFSET_IDX]], -2
@@ -2876,12 +2882,6 @@ define i32 @sink_into_replication_region_multiple(ptr %x, i32 %y) {
; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = add i32 [[OFFSET_IDX]], -5
; UNROLL-NO-IC-NEXT: [[TMP8:%.*]] = add i32 [[OFFSET_IDX]], -6
; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = add i32 [[OFFSET_IDX]], -7
-; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i32> poison, i32 [[INDEX]], i64 0
-; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT2]], <4 x i32> poison, <4 x i32> zeroinitializer
-; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = add nuw <4 x i32> [[BROADCAST_SPLAT3]], <i32 0, i32 1, i32 2, i32 3>
-; UNROLL-NO-IC-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i32> [[BROADCAST_SPLAT3]], <i32 4, i32 5, i32 6, i32 7>
-; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = icmp ule <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; UNROLL-NO-IC-NEXT: [[TMP11:%.*]] = icmp ule <4 x i32> [[STEP_ADD]], [[BROADCAST_SPLAT]]
; UNROLL-NO-IC-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP10]], i64 0
; UNROLL-NO-IC-NEXT: br i1 [[TMP12]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
; UNROLL-NO-IC: [[PRED_UDIV_IF]]:
@@ -3035,12 +3035,12 @@ define i32 @sink_into_replication_region_multiple(ptr %x, i32 %y) {
; UNROLL-NO-VF-NEXT: [[VECTOR_RECUR:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[PRED_STORE_CONTINUE6]] ]
; UNROLL-NO-VF-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[PRED_STORE_CONTINUE6]] ]
; UNROLL-NO-VF-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[PRED_STORE_CONTINUE6]] ]
-; UNROLL-NO-VF-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]]
-; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = add i32 [[OFFSET_IDX]], -1
; UNROLL-NO-VF-NEXT: [[VEC_IV:%.*]] = add i32 [[INDEX]], 0
; UNROLL-NO-VF-NEXT: [[VEC_IV2:%.*]] = add i32 [[INDEX]], 1
; UNROLL-NO-VF-NEXT: [[TMP4:%.*]] = icmp ule i32 [[VEC_IV]], [[TRIP_COUNT_MINUS_1]]
; UNROLL-NO-VF-NEXT: [[TMP5:%.*]] = icmp ule i32 [[VEC_IV2]], [[TRIP_COUNT_MINUS_1]]
+; UNROLL-NO-VF-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]]
+; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = add i32 [[OFFSET_IDX]], -1
; UNROLL-NO-VF-NEXT: br i1 [[TMP4]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
; UNROLL-NO-VF: [[PRED_UDIV_IF]]:
; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = udiv i32 219220132, [[OFFSET_IDX]]
@@ -3098,11 +3098,11 @@ define i32 @sink_into_replication_region_multiple(ptr %x, i32 %y) {
; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[PRED_STORE_CONTINUE12]] ]
; SINK-AFTER-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[PRED_STORE_CONTINUE12]] ]
; SINK-AFTER-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE12]] ]
+; SINK-AFTER-NEXT: [[TMP6:%.*]] = icmp ule <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; SINK-AFTER-NEXT: [[OFFSET_IDX:%.*]] = sub i32 [[Y]], [[INDEX]]
; SINK-AFTER-NEXT: [[TMP3:%.*]] = add i32 [[OFFSET_IDX]], -1
; SINK-AFTER-NEXT: [[TMP4:%.*]] = add i32 [[OFFSET_IDX]], -2
; SINK-AFTER-NEXT: [[TMP5:%.*]] = add i32 [[OFFSET_IDX]], -3
-; SINK-AFTER-NEXT: [[TMP6:%.*]] = icmp ule <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
; SINK-AFTER-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i64 0
; SINK-AFTER-NEXT: br i1 [[TMP7]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
; SINK-AFTER: [[PRED_UDIV_IF]]:
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll
index 664263ae78bdc..cc1d4b2064565 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-fold-tail.ll
@@ -85,8 +85,8 @@ define i64 @select_decreasing_induction_icmp_non_const_start(ptr %a, ptr %b, i64
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ splat (i64 9223372036854775807), %[[VECTOR_PH]] ], [ [[TMP52:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP50:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT8:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[N]], [[INDEX]]
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i64 [[N]], [[INDEX]]
; CHECK-NEXT: [[TMP38:%.*]] = add nsw i64 [[OFFSET_IDX]], -1
; CHECK-NEXT: [[TMP39:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP38]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[TMP39]], i64 -3
diff --git a/llvm/test/Transforms/LoopVectorize/optsize.ll b/llvm/test/Transforms/LoopVectorize/optsize.ll
index d54920acc5f61..2fe02c62c7790 100644
--- a/llvm/test/Transforms/LoopVectorize/optsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/optsize.ll
@@ -483,13 +483,13 @@ define i32 @pr45526() optsize {
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)
; CHECK-NEXT: [[TMP1]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 512
; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; CHECK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.v4i1(<4 x i1> [[TMP0]], i1 false)
; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i32 [[FIRST_INACTIVE_LANE]], 1
@@ -508,13 +508,13 @@ define i32 @pr45526() optsize {
; PGSO-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; PGSO-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; PGSO-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
+; PGSO-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)
; PGSO-NEXT: [[TMP1]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)
; PGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; PGSO-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
; PGSO-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 512
; PGSO-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; PGSO: [[MIDDLE_BLOCK]]:
-; PGSO-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)
; PGSO-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; PGSO-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.v4i1(<4 x i1> [[TMP0]], i1 false)
; PGSO-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i32 [[FIRST_INACTIVE_LANE]], 1
@@ -533,13 +533,13 @@ define i32 @pr45526() optsize {
; NPGSO-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NPGSO-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NPGSO-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
+; NPGSO-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)
; NPGSO-NEXT: [[TMP1]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)
; NPGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; NPGSO-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
; NPGSO-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 512
; NPGSO-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; NPGSO: [[MIDDLE_BLOCK]]:
-; NPGSO-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)
; NPGSO-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; NPGSO-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.v4i1(<4 x i1> [[TMP0]], i1 false)
; NPGSO-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i32 [[FIRST_INACTIVE_LANE]], 1
@@ -574,13 +574,13 @@ define i32 @pr45526_pgso() !prof !14 {
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)
; CHECK-NEXT: [[TMP1]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 512
; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; CHECK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.v4i1(<4 x i1> [[TMP0]], i1 false)
; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i32 [[FIRST_INACTIVE_LANE]], 1
@@ -599,13 +599,13 @@ define i32 @pr45526_pgso() !prof !14 {
; PGSO-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; PGSO-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; PGSO-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
+; PGSO-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)
; PGSO-NEXT: [[TMP1]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)
; PGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; PGSO-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
; PGSO-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 512
; PGSO-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; PGSO: [[MIDDLE_BLOCK]]:
-; PGSO-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)
; PGSO-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; PGSO-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.v4i1(<4 x i1> [[TMP0]], i1 false)
; PGSO-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i32 [[FIRST_INACTIVE_LANE]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/pr51614-fold-tail-by-masking.ll b/llvm/test/Transforms/LoopVectorize/pr51614-fold-tail-by-masking.ll
index d4fb13d148fa8..3385cf58e3e61 100644
--- a/llvm/test/Transforms/LoopVectorize/pr51614-fold-tail-by-masking.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr51614-fold-tail-by-masking.ll
@@ -18,9 +18,9 @@ define i16 @reverse_interleave_load_fold_mask() optsize {
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE4:%.*]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i16> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP25:%.*]], [[PRED_LOAD_CONTINUE4]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i8> [ <i8 0, i8 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE4]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <2 x i8> [[VEC_IND]], splat (i8 40)
; CHECK-NEXT: [[TMP0:%.*]] = trunc i32 [[INDEX]] to i16
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = sub i16 41, [[TMP0]]
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <2 x i8> [[VEC_IND]], splat (i8 40)
; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
; CHECK: pred.load.if:
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-order.ll b/llvm/test/Transforms/LoopVectorize/reduction-order.ll
index 6e3a1951d15fb..7200d6f833284 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-order.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-order.ll
@@ -17,6 +17,7 @@ define i32 @foo() !prof !1 {
; CHECK-NEXT: [[VEC_PHI_1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[ADD_5:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI_2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[ADD_3:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 9)
; CHECK-NEXT: [[ADD_3]] = add <4 x i32> splat (i32 3), [[VEC_PHI_2]]
; CHECK-NEXT: [[ADD_5]] = add <4 x i32> [[VEC_PHI_1]], splat (i32 5)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
@@ -24,7 +25,6 @@ define i32 @foo() !prof !1 {
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 12
; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 9)
; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[ADD_5]], <4 x i32> [[VEC_PHI_1]]
; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP0]], <4 x i32> [[ADD_3]], <4 x i32> [[VEC_PHI_2]]
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
diff --git a/llvm/test/Transforms/LoopVectorize/select-reduction.ll b/llvm/test/Transforms/LoopVectorize/select-reduction.ll
index c69ef5299fe6c..44aaec165762f 100644
--- a/llvm/test/Transforms/LoopVectorize/select-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-reduction.ll
@@ -25,16 +25,16 @@ define i32 @test(i64 %N, i32 %x) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP1]] = select <4 x i1> [[TMP0]], <4 x i1> splat (i1 true), <4 x i1> [[VEC_PHI]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP1]] = select <4 x i1> [[TMP6]], <4 x i1> splat (i1 true), <4 x i1> [[VEC_PHI]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <4 x i64> [[VEC_IV]], splat (i64 4)
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP1]])
-; CHECK-NEXT: [[TMP6:%.*]] = freeze i1 [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = select i1 [[TMP6]], i32 10, i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = freeze i1 [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = select i1 [[TMP7]], i32 10, i32 0
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: exit.loopexit:
; CHECK-NEXT: br label [[EXIT]]
diff --git a/llvm/test/Transforms/LoopVectorize/store-reduction-results-in-tail-folded-loop.ll b/llvm/test/Transforms/LoopVectorize/store-reduction-results-in-tail-folded-loop.ll
index 024cb0a4c6d4c..e29d808c403f2 100644
--- a/llvm/test/Transforms/LoopVectorize/store-reduction-results-in-tail-folded-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/store-reduction-results-in-tail-folded-loop.ll
@@ -27,6 +27,7 @@ define void @pr75298_store_reduction_value_in_folded_loop(i64 %iv.start) optsize
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IV:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr @c, align 4
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i32> poison, i32 [[TMP2]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT3]], <4 x i32> poison, <4 x i32> zeroinitializer
@@ -36,7 +37,6 @@ define void @pr75298_store_reduction_value_in_folded_loop(i64 %iv.start) optsize
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP7:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP7]], <4 x i32> [[TMP3]], <4 x i32> [[VEC_PHI]]
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> [[TMP4]])
; CHECK-NEXT: store i32 [[TMP6]], ptr @a, align 4
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-div.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-div.ll
index 8767d44723e61..7f6d19fbe0230 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-div.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-div.ll
@@ -13,8 +13,8 @@ define void @test_sdiv_variant_divisor_induction(ptr noalias %a, ptr noalias %c)
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND1:%.*]] = phi <2 x i64> [ <i64 1, i64 2>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND2:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <2 x i16> [[VEC_IND2]], splat (i16 1024)
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[A]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[TMP12:%.*]] = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr align 4 [[TMP4]], <2 x i1> [[TMP2]], <2 x i64> poison)
; CHECK-NEXT: [[TMP13:%.*]] = call <2 x i64> @llvm.masked.sdiv.v2i64(<2 x i64> [[TMP12]], <2 x i64> [[VEC_IND1]], <2 x i1> [[TMP2]])
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-iv-outside-user.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-iv-outside-user.ll
index e2e936216ea8f..b23019fe77e8f 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-iv-outside-user.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-iv-outside-user.ll
@@ -28,13 +28,13 @@ define i32 @f(ptr noalias %p, i32 %start, i32 %step, i32 %n) {
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IV:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt <4 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT6]]
; CHECK-NEXT: [[VEC_IND_NEXT8]] = add nuw <4 x i32> [[VEC_IV]], splat (i32 4)
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt <4 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
; CHECK-NEXT: [[IV2_LCSSA:%.*]] = extractelement <4 x i32> [[VEC_IND]], i64 [[LAST_ACTIVE_LANE]]
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-replicate-region.ll
index 003e100db1506..d8f5b8c316438 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-replicate-region.ll
@@ -17,10 +17,10 @@ define void @f(ptr %p, i64 %n) {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE12:.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE12]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 1
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 2
; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT: [[TMP3:%.*]] = icmp ule <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[P]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP0]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP1]]
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
index aa9a6a15838f4..ea410af12d5ff 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
@@ -82,6 +82,13 @@ define void @VF1-VPWidenCanonicalIVRecipeExe(ptr %ptr1) {
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE12:%.*]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i64 [[INDEX]], 14
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ule i64 [[TMP7]], 14
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ule i64 [[TMP9]], 14
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ule i64 [[TMP10]], 14
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 3
; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 8
; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 16
@@ -90,13 +97,6 @@ define void @VF1-VPWidenCanonicalIVRecipeExe(ptr %ptr1) {
; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP4]]
; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP5]]
; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[PTR1]], i64 [[TMP6]]
-; CHECK-NEXT: [[VEC_IV4:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[VEC_IV5:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IV6:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i64 [[INDEX]], 14
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ule i64 [[VEC_IV4]], 14
-; CHECK-NEXT: [[TMP2:%.*]] = icmp ule i64 [[VEC_IV5]], 14
-; CHECK-NEXT: [[TMP3:%.*]] = icmp ule i64 [[VEC_IV6]], 14
; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; CHECK: pred.store.if:
; CHECK-NEXT: store double 0.000000e+00, ptr [[NEXT_GEP]], align 8
@@ -160,15 +160,15 @@ define i64 @live_out_scalar_vf(i64 %n) {
; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <4 x i64> [[VEC_IND]], splat (i64 4)
; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add nuw <4 x i64> [[STEP_ADD]], splat (i64 4)
; CHECK-NEXT: [[STEP_ADD_3]] = add nuw <4 x i64> [[STEP_ADD_2]], splat (i64 4)
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt <4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt <4 x i64> [[STEP_ADD_2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt <4 x i64> [[STEP_ADD_3]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 16
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[STEP_ADD_3]], splat (i64 4)
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt <4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt <4 x i64> [[STEP_ADD_2]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt <4 x i64> [[STEP_ADD_3]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> [[VEC_IND]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; CHECK-NEXT: [[TMP29:%.*]] = shufflevector <4 x i64> [[VEC_IND]], <4 x i64> [[STEP_ADD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; CHECK-NEXT: [[TMP31:%.*]] = shufflevector <4 x i64> [[STEP_ADD]], <4 x i64> [[STEP_ADD_2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
diff --git a/llvm/test/Transforms/LoopVectorize/use-scalar-epilogue-if-tp-fails.ll b/llvm/test/Transforms/LoopVectorize/use-scalar-epilogue-if-tp-fails.ll
index 6045451ba587e..33a772285e423 100644
--- a/llvm/test/Transforms/LoopVectorize/use-scalar-epilogue-if-tp-fails.ll
+++ b/llvm/test/Transforms/LoopVectorize/use-scalar-epilogue-if-tp-fails.ll
@@ -108,6 +108,7 @@ define void @metadata(ptr nocapture readonly %ptr, i32 %size, ptr %pos) {
; FORCED-TF: vector.body:
; FORCED-TF-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE11:%.*]] ]
; FORCED-TF-NEXT: [[VEC_IV:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE11]] ]
+; FORCED-TF-NEXT: [[TMP8:%.*]] = icmp ule <4 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]]
; FORCED-TF-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1
; FORCED-TF-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 2
; FORCED-TF-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 3
@@ -115,7 +116,6 @@ define void @metadata(ptr nocapture readonly %ptr, i32 %size, ptr %pos) {
; FORCED-TF-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[PTR]], i32 [[TMP1]]
; FORCED-TF-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[PTR]], i32 [[TMP2]]
; FORCED-TF-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[PTR]], i32 [[TMP3]]
-; FORCED-TF-NEXT: [[TMP8:%.*]] = icmp ule <4 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]]
; FORCED-TF-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP8]], i64 0
; FORCED-TF-NEXT: br i1 [[TMP17]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; FORCED-TF: pred.store.if:
@@ -174,6 +174,7 @@ define void @metadata(ptr nocapture readonly %ptr, i32 %size, ptr %pos) {
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE11:%.*]] ]
; CHECK-NEXT: [[VEC_IV:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE11]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <4 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 3
@@ -181,7 +182,6 @@ define void @metadata(ptr nocapture readonly %ptr, i32 %size, ptr %pos) {
; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[PTR]], i32 [[TMP1]]
; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[PTR]], i32 [[TMP2]]
; CHECK-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[PTR]], i32 [[TMP3]]
-; CHECK-NEXT: [[TMP8:%.*]] = icmp ule <4 x i32> [[VEC_IV]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i1> [[TMP8]], i64 0
; CHECK-NEXT: br i1 [[TMP17]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; CHECK: pred.store.if:
diff --git a/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-tail-folding.ll
index bdaf9b37da10c..6fb47cb2bdf07 100644
--- a/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-tail-folding.ll
@@ -14,9 +14,9 @@ define i32 @last_active_lane_live_out(i32 %x) {
; CHECK-NEXT: [[TMP0:%.*]] = mul <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[BROADCAST_SPLAT]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 2)
; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 2)
; CHECK-NEXT: [[TMP1:%.*]] = xor <4 x i1> [[ACTIVE_LANE_MASK]], splat (i1 true)
; CHECK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP1]], i1 false)
; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
diff --git a/llvm/test/Transforms/PhaseOrdering/ARM/arm_add_q7.ll b/llvm/test/Transforms/PhaseOrdering/ARM/arm_add_q7.ll
index 811957c7ea5d3..25b2823fd364c 100644
--- a/llvm/test/Transforms/PhaseOrdering/ARM/arm_add_q7.ll
+++ b/llvm/test/Transforms/PhaseOrdering/ARM/arm_add_q7.ll
@@ -19,10 +19,10 @@ define dso_local void @arm_add_q7(ptr %pSrcA, ptr %pSrcB, ptr noalias %pDst, i32
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = tail call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[BLOCKSIZE]])
; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PSRCA:%.*]], i32 [[INDEX]]
; CHECK-NEXT: [[NEXT_GEP14:%.*]] = getelementptr i8, ptr [[PDST:%.*]], i32 [[INDEX]]
; CHECK-NEXT: [[NEXT_GEP15:%.*]] = getelementptr i8, ptr [[PSRCB:%.*]], i32 [[INDEX]]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = tail call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 [[BLOCKSIZE]])
; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = tail call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; CHECK-NEXT: [[WIDE_MASKED_LOAD16:%.*]] = tail call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 [[NEXT_GEP15]], <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
; CHECK-NEXT: [[TMP2:%.*]] = tail call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> [[WIDE_MASKED_LOAD]], <16 x i8> [[WIDE_MASKED_LOAD16]])
>From d7813331eecd5e60dba214e6a5ab542884af9f84 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Mon, 29 Jun 2026 12:34:02 +0100
Subject: [PATCH 3/3] !fixup check for CanIV directly instead of regionvalue.
---
llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 77a33339eb5f9..4279336655333 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -248,8 +248,8 @@ SmallVector<VPRegisterUsage, 8> llvm::calculateRegisterUsageForPlan(
match(VPV, m_ExtractLastPart(m_VPValue())))
continue;
- if (VFs[J].isScalar() ||
- isa<VPRegionValue, VPReplicateRecipe, VPDerivedIVRecipe,
+ if (VFs[J].isScalar() || VPV == CanIV ||
+ isa<VPReplicateRecipe, VPDerivedIVRecipe,
VPCurrentIterationPHIRecipe, VPScalarIVStepsRecipe>(VPV) ||
(isa<VPInstruction>(VPV) && vputils::onlyScalarValuesUsed(VPV)) ||
(isa<VPReductionPHIRecipe>(VPV) &&
More information about the llvm-commits
mailing list