[llvm] [LV] Vectorize early exit loops with stores using masking (PR #178454)
Graham Hunter via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 8 09:30:26 PDT 2026
https://github.com/huntergr-arm updated https://github.com/llvm/llvm-project/pull/178454
>From 3bddc4acbc3db3648c36d5c04cadc2b4c3cb5af3 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Tue, 1 Jul 2025 13:08:48 +0000
Subject: [PATCH 1/3] [LV] Move condition instructions for uncountable exit
loops with stores
This is the core transformation for supporting uncountable exit loops with stores.
In order to avoid writing to memory that the scalar loop wouldn't have, we must
figure out which lane (if any) an exit occurs on, then create a mask for it. If
other memory operations (loads or stores) occur before the loads contributing to
the exit condition, then we must move the load and any other instructions needed
to determine whether the exit occurs so the mask can be created.
Even if a backend wants a different transformation, it will still need to be able
to move/copy the instructions needed for the exit condition.
---
.../Vectorize/LoopVectorizationLegality.cpp | 29 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 40 ++-
llvm/lib/Transforms/Vectorize/VPlan.cpp | 2 +
llvm/lib/Transforms/Vectorize/VPlan.h | 8 +-
.../Vectorize/VPlanConstruction.cpp | 11 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 199 ++++++++++-
.../Transforms/Vectorize/VPlanTransforms.h | 2 +-
.../AArch64/early_exit_with_stores.ll | 265 +++++++++++++--
.../RISCV/early_exit_with_stores.ll | 281 ++++++++++++---
.../VPlan/early_exit_with_stores_vplan.ll | 194 ++++++++++-
.../early_exit_store_legality.ll | 23 +-
.../LoopVectorize/early_exit_with_stores.ll | 319 ++++++++++++++----
12 files changed, 1194 insertions(+), 179 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 07e6780d9e7ba..b42937f867e5b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1911,7 +1911,11 @@ bool LoopVectorizationLegality::canUncountableExitConditionLoadBeMoved(
if (&I == Load)
continue;
- if (I.mayWriteToMemory()) {
+ if (I.mayReadOrWriteMemory()) {
+ // We need to mask all other memory ops.
+ ConditionallyExecutedOps.insert(&I);
+ if (isa<LoadInst>(&I))
+ continue;
if (auto *SI = dyn_cast<StoreInst>(&I)) {
AliasResult AR = AA->alias(Ptr, SI->getPointerOperand());
if (AR == AliasResult::NoAlias)
@@ -2017,12 +2021,23 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
return false;
}
- // Bail out for ReadWrite loops with uncountable exits for now.
- if (UncountableExitType == UncountableExitTrait::ReadWrite) {
- reportVectorizationFailure(
- "Writes to memory unsupported in early exit loops",
- "Cannot vectorize early exit loop with writes to memory",
- "WritesInEarlyExitLoop", ORE, TheLoop);
+ // TODO: Remove this restriction once we're sure it's safe to do so.
+ // Handling stores to invariant addresses will be slightly different
+ // based on the vectorization style chosen. If we bail out to a scalar
+ // tail before executing any lane that would take the uncountable exit,
+ // then the store that occurs in the scalar loop would suffice.
+ //
+ // If we instead handle the lane taking the uncountable exit within the
+ // vectorized loop, then we will have to ensure that we extract the
+ // last active lane at that point in the loop instead of the last lane
+ // of the vector before performing a scalar store.
+ if (UncountableExitType == UncountableExitTrait::ReadWrite &&
+ !LAI->getStoresToInvariantAddresses().empty()) {
+ LLVM_DEBUG(dbgs() << "LV: Cannot vectorize early exit loops with stores to "
+ "loop-invariant addresses\n");
+ reportVectorizationFailure("Cannot vectorize early exit loops with stores "
+ "to loop-invariant addresses",
+ "LoopInvariantStoresInEELoop", ORE, TheLoop);
return false;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e907b5621b817..559949c03286d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -418,6 +418,12 @@ static cl::opt<bool> EnableEarlyExitVectorization(
cl::desc(
"Enable vectorization of early exit loops with uncountable exits."));
+static cl::opt<bool> EnableEarlyExitVectorizationWithSideEffects(
+ "enable-early-exit-vectorization-with-side-effects", cl::init(false),
+ cl::Hidden,
+ cl::desc("Enable vectorization of early exit loops with uncountable exits "
+ "and side effects"));
+
static cl::opt<bool> ConsiderRegPressure(
"vectorizer-consider-reg-pressure", cl::init(false), cl::Hidden,
cl::desc("Discard VFs if their register pressure is too high."));
@@ -2763,8 +2769,9 @@ bool LoopVectorizationCostModel::isPredicatedInst(Instruction *I) const {
if (Legal->blockNeedsPredication(I->getParent()))
return true;
- // If we're not folding the tail by masking, predication is unnecessary.
- if (!foldTailByMasking())
+ // If we're not folding the tail by masking or vectorizing a loop with
+ // uncountable exits and side effects, predication is unnecessary.
+ if (!foldTailByMasking() && !Legal->hasUncountableExitWithSideEffects())
return false;
// All that remain are instructions with side-effects originally executed in
@@ -4334,6 +4341,11 @@ bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
if (OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch())
return false;
+ // Loops with uncountable exits and side effects (e.g. stores) are not yet
+ // supported for epilogue vectorization.
+ if (Legal->hasUncountableExitWithSideEffects())
+ return false;
+
return true;
}
@@ -7971,7 +7983,12 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
if (!VPlanTransforms::handleEarlyExits(*VPlan0, EEStyle, OrigLoop, PSE, *DT,
Legal->getAssumptionCache()))
return;
- VPlanTransforms::addMiddleCheck(*VPlan0, CM.foldTailByMasking());
+
+ // If we're handling uncountable exits in the scalar tail after a vector
+ // loop with an in-loop mask, then the middle check has already been
+ // created to compare against the actual number of lanes executed.
+ if (EEStyle != UncountableExitStyle::MaskedHandleExitInScalarLoop)
+ VPlanTransforms::addMiddleCheck(*VPlan0, CM.foldTailByMasking());
RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::createLoopRegions, *VPlan0);
if (CM.foldTailByMasking())
RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::foldTailByMasking, *VPlan0);
@@ -9329,6 +9346,14 @@ bool LoopVectorizePass::processLoop(Loop *L) {
"UncountableEarlyExitLoopsDisabled", ORE, L);
return false;
}
+ if (LVL.hasUncountableExitWithSideEffects() &&
+ !EnableEarlyExitVectorizationWithSideEffects) {
+ reportVectorizationFailure("Auto-vectorization of loops with uncountable "
+ "early exit and side effects is not enabled",
+ "UncountableEarlyExitSideEffectLoopsDisabled",
+ ORE, L);
+ return false;
+ }
}
// Entrance to the VPlan-native vectorization path. Outer loops are processed
@@ -9570,6 +9595,15 @@ bool LoopVectorizePass::processLoop(Loop *L) {
// Override IC if user provided an interleave count.
IC = UserIC > 0 ? UserIC : IC;
+ // FIXME: Enable interleaving for EE-with-side-effects.
+ if (InterleaveLoop && LVL.hasUncountableExitWithSideEffects()) {
+ LLVM_DEBUG(dbgs() << "LV: Not interleaving due to EE with side effects.\n");
+ IntDiagMsg = {"EEWithSideEffectsPreventsInterleaving",
+ "Unable to interleave due to early exit with side effects."};
+ InterleaveLoop = false;
+ IC = 1;
+ }
+
// Emit diagnostic messages, if any.
const char *VAPassName = Hints.vectorizeAnalysisPassName();
if (!VectorizeLoop && !InterleaveLoop) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 3ba5ac582fa9a..82099e49e6371 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1284,6 +1284,8 @@ VPlan *VPlan::duplicate() {
NewPlan->ExitBlocks.push_back(cast<VPIRBasicBlock>(VPB));
}
+ NewPlan->HasEarlyExitWithSideEffects = HasEarlyExitWithSideEffects;
+
return NewPlan;
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 21016f84d48a7..4b21d77aa9ba1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -4625,6 +4625,8 @@ class VPlan {
/// VPlan is destroyed.
SmallVector<VPBlockBase *> CreatedBlocks;
+ bool HasEarlyExitWithSideEffects = false;
+
/// Construct a VPlan with \p Entry to the plan and with \p ScalarHeader
/// wrapping the original header of the scalar loop.
VPlan(VPBasicBlock *Entry, VPIRBasicBlock *ScalarHeader)
@@ -4949,9 +4951,13 @@ class VPlan {
return count_if(ExitBlocks,
[](VPIRBasicBlock *EB) { return EB->hasPredecessors(); }) >
1 ||
- (ExitBlocks.size() == 1 && ExitBlocks[0]->getNumPredecessors() > 1);
+ (ExitBlocks.size() == 1 &&
+ ExitBlocks[0]->getNumPredecessors() > 1) ||
+ HasEarlyExitWithSideEffects;
}
+ void setHasEarlyExitWithSideEffects() { HasEarlyExitWithSideEffects = true; }
+
/// Returns true if the scalar tail may execute after the vector loop. Note
/// that this relies on unneeded branches to the scalar tail loop being
/// removed.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index c1df51841076f..91039d1d9b52b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1000,12 +1000,17 @@ bool VPlanTransforms::handleEarlyExits(VPlan &Plan, UncountableExitStyle Style,
// here from handleUncountableEarlyExits, but we need to improve
// detection of recipes which may write to memory.
if (Style != UncountableExitStyle::NoUncountableExit) {
- if (!areAllLoadsDereferenceable(HeaderVPBB, MiddleVPBB, TheLoop, PSE, DT,
+ // Legality is still performing the dereferenceability check for
+ // uncountable exit loops with stores; only the loads that contribute to
+ // the condition must be verified, since every other memory operation will
+ // be masked according to the exit condition.
+ if (Style == UncountableExitStyle::ReadOnly &&
+ !areAllLoadsDereferenceable(HeaderVPBB, MiddleVPBB, TheLoop, PSE, DT,
AC))
return false;
// TODO: Check target preference for style.
- handleUncountableEarlyExits(Plan, HeaderVPBB, LatchVPBB, MiddleVPBB, Style);
- return true;
+ return handleUncountableEarlyExits(Plan, HeaderVPBB, LatchVPBB, MiddleVPBB,
+ Style);
}
// Disconnect countable early exits from the loop, leaving it with a single
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 5fbdb2aa98d9f..2aba522a773bc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4157,17 +4157,181 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
R->eraseFromParent();
}
-void VPlanTransforms::handleUncountableEarlyExits(VPlan &Plan,
+struct EarlyExitInfo {
+ VPBasicBlock *EarlyExitingVPBB;
+ VPIRBasicBlock *EarlyExitVPBB;
+ VPValue *CondToExit;
+};
+
+/// Update \p Plan to mask memory operations in the loop based on whether the
+/// early exit is taken or not.
+static bool handleUncountableExitsWithSideEffects(
+ VPlan &Plan, SmallVectorImpl<EarlyExitInfo> &Exits,
+ VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB,
+ VPBasicBlock *MiddleVPBB) {
+ if (Plan.hasScalarVFOnly())
+ return false;
+
+ Plan.setHasEarlyExitWithSideEffects();
+
+ // Disconnect early exiting blocks from successors, remove branches. We
+ // currently don't support multiple uses for recipes involved in creating
+ // the uncountable exit condition.
+ for (auto &Exit : Exits) {
+ if (Exit.EarlyExitingVPBB == LatchVPBB)
+ continue;
+
+ for (VPRecipeBase &R : Exit.EarlyExitVPBB->phis())
+ cast<VPIRPhi>(&R)->removeIncomingValueFor(Exit.EarlyExitingVPBB);
+ Exit.EarlyExitingVPBB->getTerminator()->eraseFromParent();
+ VPBlockUtils::disconnectBlocks(Exit.EarlyExitingVPBB, Exit.EarlyExitVPBB);
+ }
+
+ // We can abandon a vplan entirely if we return false here, so we shouldn't
+ // crash if some earlier assumptions on scalar IR don't hold for the vplan
+ // version of the loop.
+ SmallVector<VPInstruction *, 2> GEPs;
+ SmallVector<VPInstruction *, 8> ConditionRecipes;
+
+ std::optional<VPValue *> Cond =
+ vputils::getRecipesForUncountableExit(ConditionRecipes, GEPs, LatchVPBB);
+ if (!Cond)
+ return false;
+
+ // Find load contributing to condition.
+ VPRecipeBase *CondLoad = nullptr;
+ for (auto *Recipe : ConditionRecipes) {
+ if (match(Recipe, m_VPInstruction<Instruction::Load>(m_VPValue()))) {
+ // TODO: Support more than one load. Needs legality updates too.
+ assert(CondLoad == nullptr && "Too many condition loads\n");
+ CondLoad = Recipe;
+ }
+ }
+ assert(CondLoad && "Couldn't find load\n");
+
+ // Check GEPs to see if we can link them to a widen IV recipe with a step of
+ // 1; we're only interested in contiguous accesses for the condition load
+ // right now.
+ using namespace llvm::VPlanPatternMatch;
+
+ for (auto *GEP : GEPs) {
+ VPValue *MaybeIV = nullptr;
+ if (!match(GEP, m_VPInstruction<Instruction::GetElementPtr>(
+ m_LiveIn(), m_VPValue(MaybeIV))))
+ return false;
+
+ auto *WIV = dyn_cast<VPWidenInductionRecipe>(MaybeIV);
+ if (!WIV)
+ return false;
+
+ auto *ConstStart = dyn_cast<VPConstantInt>(WIV->getStartValue());
+ if (!ConstStart || !ConstStart->isZero())
+ return false;
+
+ auto *ConstStep = dyn_cast<VPConstantInt>(WIV->getStepValue());
+ if (!ConstStep || !ConstStep->isOne())
+ return false;
+ }
+
+ // Find an insertion point. Default to the end of the header (we haven't
+ // performed if-conversion yet, so there will likely be more than one block),
+ // but override if we find a memory op that needs masking before the
+ // condition load.
+ auto InsertIt = HeaderVPBB->end();
+ VPRecipeBase *CondR = (*Cond)->getDefiningRecipe();
+ bool CondMoveNeeded = CondR->getParent() != HeaderVPBB;
+ for (VPRecipeBase &R : make_early_inc_range(*HeaderVPBB)) {
+ if (&R == CondLoad)
+ continue;
+
+ if (match(&R, m_CombineOr(m_VPInstruction<Instruction::Load>(m_VPValue()),
+ m_VPInstruction<Instruction::Store>(
+ m_VPValue(), m_VPValue())))) {
+ VPDominatorTree VPDT(Plan);
+ if (!VPDT.properlyDominates(CondR, &R)) {
+ CondMoveNeeded = true;
+ InsertIt = R.getIterator();
+ }
+ break;
+ }
+ }
+
+ // If another memory operation would take place before the comparison to
+ // determine whether to exit early or the comparison doesn't take place in
+ // the header, move the comparison (and supporting recipes).
+ if (CondMoveNeeded) {
+ for (auto *Recipe : reverse(ConditionRecipes))
+ Recipe->moveBefore(*HeaderVPBB, InsertIt);
+
+ CondR->moveBefore(*HeaderVPBB, InsertIt);
+ }
+
+ // Create a mask to represent all lanes that fully execute in the vector loop,
+ // stopping short of any early exit.
+ VPBuilder MaskBuilder(HeaderVPBB, InsertIt);
+ VPValue *FirstActive =
+ MaskBuilder.createNaryOp(VPInstruction::FirstActiveLane, *Cond);
+ VPCanonicalIVPHIRecipe *IV =
+ dyn_cast<VPCanonicalIVPHIRecipe>(&HeaderVPBB->front());
+ VPValue *ALMMultiplier = Plan.getConstantInt(IV->getScalarType(), 1);
+ VPValue *Zero = Plan.getConstantInt(IV->getScalarType(), 0);
+ VPValue *Mask = MaskBuilder.createNaryOp(VPInstruction::ActiveLaneMask,
+ {Zero, FirstActive, ALMMultiplier},
+ nullptr, "uncountable.exit.mask");
+
+ // Convert all other memory operations to use the mask.
+ ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
+ HeaderVPBB);
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
+ for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+ if (&R == CondLoad)
+ continue;
+
+ if (match(&R, m_CombineOr(m_VPInstruction<Instruction::Load>(m_VPValue()),
+ m_VPInstruction<Instruction::Store>(
+ m_VPValue(), m_VPValue()))))
+ cast<VPInstruction>(&R)->addMask(Mask);
+ }
+ }
+
+ // Update middle block branch to compare (IV + however many lanes were active)
+ // against the full trip count, since we may be exiting the vector loop early.
+ // If we didn't take an early exit, we should get the equivalent of VF from
+ // the cttz.elts.
+ VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->end());
+ // TODO: Create popcount of mask another way... zext + reduce_add?
+ // For SVE at least we want to be able to fold away the
+ // brkb+cntp+whilelo in the loop to just brkb. We need a cntp
+ // outside the loop instead... maybe copy in CodeGenPrepare?
+ VPTypeAnalysis TypeInfo(Plan);
+
+ VPValue *ExitIV = MiddleBuilder.createAdd(IV, FirstActive);
+ VPValue *FullTC =
+ MiddleBuilder.createICmp(CmpInst::ICMP_EQ, ExitIV, Plan.getTripCount());
+ MiddleBuilder.createNaryOp(VPInstruction::BranchOnCond, {FullTC});
+
+ // Update resume phi in scalar.ph...
+ VPBasicBlock *ScalarPH = Plan.getScalarPreheader();
+ auto Phis = ScalarPH->phis();
+ bool PhiFound = false;
+ for (auto &Phi : Phis) {
+ // TODO: Handle more than one Phi; re-derive from IV.
+ // TODO: Handle reductions, one day...
+ if (PhiFound)
+ return false;
+ PhiFound = true;
+ VPPhi *ContinueIV = cast<VPPhi>(&Phi);
+ ContinueIV->setOperand(0, ExitIV);
+ }
+
+ return true;
+}
+
+bool VPlanTransforms::handleUncountableEarlyExits(VPlan &Plan,
VPBasicBlock *HeaderVPBB,
VPBasicBlock *LatchVPBB,
VPBasicBlock *MiddleVPBB,
UncountableExitStyle Style) {
- struct EarlyExitInfo {
- VPBasicBlock *EarlyExitingVPBB;
- VPIRBasicBlock *EarlyExitVPBB;
- VPValue *CondToExit;
- };
-
VPDominatorTree VPDT(Plan);
VPBuilder Builder(LatchVPBB->getTerminator());
SmallVector<EarlyExitInfo> Exits;
@@ -4239,8 +4403,23 @@ void VPlanTransforms::handleUncountableEarlyExits(VPlan &Plan,
VPValue *IsAnyExitTaken =
Builder.createNaryOp(VPInstruction::AnyOf, {Combined});
- assert(Style == UncountableExitStyle::ReadOnly &&
- "Early exit store masking not implemented");
+ if (Style == UncountableExitStyle::MaskedHandleExitInScalarLoop) {
+ auto *LatchExitingBranch = cast<VPInstruction>(LatchVPBB->getTerminator());
+ assert(LatchExitingBranch->getOpcode() == VPInstruction::BranchOnCount &&
+ "Unexpected terminator");
+ auto *IsLatchExitTaken =
+ Builder.createICmp(CmpInst::ICMP_EQ, LatchExitingBranch->getOperand(0),
+ LatchExitingBranch->getOperand(1));
+
+ DebugLoc LatchDL = LatchExitingBranch->getDebugLoc();
+ LatchExitingBranch->eraseFromParent();
+ Builder.setInsertPoint(LatchVPBB);
+ VPValue *Combined =
+ Builder.createOr(IsAnyExitTaken, IsLatchExitTaken, LatchDL);
+ Builder.createNaryOp(VPInstruction::BranchOnCond, {Combined}, LatchDL);
+ return handleUncountableExitsWithSideEffects(Plan, Exits, HeaderVPBB,
+ LatchVPBB, MiddleVPBB);
+ }
// Create the vector.early.exit blocks.
SmallVector<VPBasicBlock *> VectorEarlyExitVPBBs(Exits.size());
@@ -4381,6 +4560,8 @@ void VPlanTransforms::handleUncountableEarlyExits(VPlan &Plan,
LatchVPBB->clearSuccessors();
LatchVPBB->setSuccessors({DispatchVPBB, MiddleVPBB, HeaderVPBB});
DispatchVPBB->setPredecessors({LatchVPBB});
+
+ return true;
}
/// This function tries convert extended in-loop reductions to
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 6312b823e5d33..e3841b9d3ac80 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -343,7 +343,7 @@ struct VPlanTransforms {
/// appropriate branching logic in the latch that handles early exits and the
/// latch exit condition. Multiple exits are handled with a dispatch block
/// that determines which exit to take based on lane-by-lane semantics.
- static void handleUncountableEarlyExits(VPlan &Plan, VPBasicBlock *HeaderVPBB,
+ static bool handleUncountableEarlyExits(VPlan &Plan, VPBasicBlock *HeaderVPBB,
VPBasicBlock *LatchVPBB,
VPBasicBlock *MiddleVPBB,
UncountableExitStyle Style);
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
index f91a248700694..5f5d338925be9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt -S < %s -p loop-vectorize -mattr=+sve | FileCheck %s
+; RUN: opt -S < %s -p loop-vectorize -mattr=+sve -enable-early-exit-vectorization-with-side-effects | FileCheck %s
target triple = "aarch64-unknown-linux-gnu"
@@ -52,10 +52,43 @@ loop.end:
define void @loop_contains_store_condition_load_has_single_user(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: define void @loop_contains_store_condition_load_has_single_user(
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: [[SCALAR_PH1:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
+; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP7]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
+; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[SCALAR_PH1]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
@@ -63,11 +96,11 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -253,21 +286,54 @@ define void @loop_contains_store_assumed_bounds(ptr noalias %array, ptr readonly
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[N_BYTES:%.*]] = mul nuw nsw i64 [[N]], 2
; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[PRED]], i64 2), "dereferenceable"(ptr [[PRED]], i64 [[N_BYTES]]) ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[TMP7]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], [[N]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR1]], align 2
; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
-; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR1]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -300,22 +366,56 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-LABEL: define void @loop_contains_store_to_pointer_with_no_deref_info(
; CHECK-SAME: ptr readonly align 2 dereferenceable(40) [[LOAD_ARRAY:%.*]], ptr noalias align 2 [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[LD_ADDR]], align 2
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[LD_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[TMP9]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP3]]
+; CHECK-NEXT: [[TMP10:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP10]])
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[IV]], [[TMP7]]
+; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[TMP14]], 20
+; CHECK-NEXT: br i1 [[TMP15]], label %[[EXIT:.*]], label %[[SCALAR_PH1]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[SCALAR_PH]] ]
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[LD_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[LOAD_ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[LD_ADDR1]], align 2
; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -347,21 +447,54 @@ define void @loop_contains_store_unknown_bounds(ptr align 2 dereferenceable(100)
; CHECK-LABEL: define void @loop_contains_store_unknown_bounds(
; CHECK-SAME: ptr noalias align 2 dereferenceable(100) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(100) [[PRED:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[TMP7]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], [[N]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR1]], align 2
; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
-; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR1]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -391,14 +524,47 @@ exit:
define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: define void @loop_contains_store_in_latch_block(
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: [[SCALAR_PH1:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
+; CHECK-NEXT: [[EE_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[EE_ADDR1]], align 2
+; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP5]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP6]])
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[TMP7]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[TMP7]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP6]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
+; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[SCALAR_PH1]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
@@ -406,7 +572,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -1032,10 +1198,43 @@ exit:
define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: define i32 @uncountable_exit_with_separate_exit_block(
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY1]] ]
+; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR1]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP7]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
+; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT_COUNTABLE:.*]], label %[[SCALAR_PH1]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[SCALAR_PH]] ]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
@@ -1047,7 +1246,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT_COUNTABLE:.*]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT_COUNTABLE]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: [[EXIT_COUNTABLE]]:
; CHECK-NEXT: ret i32 0
; CHECK: [[EXIT_UNCOUNTABLE]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
index 4c3d11d6c982e..138d04bc03edd 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt -S < %s -p loop-vectorize -mtriple=riscv64 -mattr=+v | FileCheck %s
+; RUN: opt -S < %s -p loop-vectorize -mtriple=riscv64 -mattr=+v -enable-early-exit-vectorization-with-side-effects | FileCheck %s
;; See ../early_exit_store_legality.ll for reasons why a particular loop doesn't
;; vectorize yet.
@@ -51,21 +51,54 @@ define void @loop_contains_store_condition_load_has_single_user(ptr dereferencea
; CHECK-LABEL: define void @loop_contains_store_condition_load_has_single_user(
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[TMP7]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
+; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH1]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[SCALAR_PH]] ]
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR1]], align 2
; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
-; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR1]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -251,21 +284,54 @@ define void @loop_contains_store_assumed_bounds(ptr noalias %array, ptr readonly
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[N_BYTES:%.*]] = mul nuw nsw i64 [[N]], 2
; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[PRED]], i64 2), "dereferenceable"(ptr [[PRED]], i64 [[N_BYTES]]) ]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[TMP7]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], [[N]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR1]], align 2
; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
-; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR1]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -298,22 +364,56 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-LABEL: define void @loop_contains_store_to_pointer_with_no_deref_info(
; CHECK-SAME: ptr readonly align 2 dereferenceable(40) [[LOAD_ARRAY:%.*]], ptr noalias align 2 [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[LD_ADDR]], align 2
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[LD_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[TMP9]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP3]]
+; CHECK-NEXT: [[TMP10:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP10]])
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[IV]], [[TMP7]]
+; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[TMP14]], 20
+; CHECK-NEXT: br i1 [[TMP15]], label %[[EXIT:.*]], label %[[SCALAR_PH1]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[SCALAR_PH]] ]
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[LD_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[LOAD_ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[LD_ADDR1]], align 2
; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -345,21 +445,54 @@ define void @loop_contains_store_unknown_bounds(ptr align 2 dereferenceable(100)
; CHECK-LABEL: define void @loop_contains_store_unknown_bounds(
; CHECK-SAME: ptr noalias align 2 dereferenceable(100) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(100) [[PRED:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[TMP7]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], [[N]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR1]], align 2
; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
-; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR1]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -435,21 +568,54 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-LABEL: define void @loop_contains_store_in_latch_block(
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
-; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP5]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP6]])
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[TMP7]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[TMP7]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[TMP6]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
+; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH1]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[SCALAR_PH]] ]
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[EE_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR1]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -1076,21 +1242,54 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-LABEL: define i32 @uncountable_exit_with_separate_exit_block(
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[TMP7]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
+; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT_COUNTABLE:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR1]], align 2
; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
-; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR1]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT_UNCOUNTABLE:.*]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT_COUNTABLE:.*]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT_COUNTABLE]], label %[[FOR_BODY1]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: [[EXIT_COUNTABLE]]:
; CHECK-NEXT: ret i32 0
; CHECK: [[EXIT_UNCOUNTABLE]]:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
index 2b17ad6c4e44a..944b879363f68 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/early_exit_with_stores_vplan.ll
@@ -1,9 +1,72 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -p loop-vectorize -force-vector-width=4 -disable-output -vplan-print-after="optimize$" %s 2>&1 | FileCheck %s
+; RUN: opt -p loop-vectorize -vplan-print-after="optimize$" -force-vector-width=4 -disable-output -enable-early-exit-vectorization-with-side-effects -force-target-supports-masked-memory-ops < %s 2>&1 | FileCheck %s -check-prefix=OPTIMIZED
; REQUIRES: asserts
define void @loop_contains_store_condition_load_has_single_user(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; OPTIMIZED-LABEL: VPlan for loop in 'loop_contains_store_condition_load_has_single_user'
+; OPTIMIZED: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; OPTIMIZED-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; OPTIMIZED-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; OPTIMIZED-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; OPTIMIZED-NEXT: Live-in ir<20> = original trip-count
+; OPTIMIZED-EMPTY:
+; OPTIMIZED-NEXT: ir-bb<entry>:
+; OPTIMIZED-NEXT: Successor(s): scalar.ph, vector.ph
+; OPTIMIZED-EMPTY:
+; OPTIMIZED-NEXT: vector.ph:
+; OPTIMIZED-NEXT: Successor(s): vector loop
+; OPTIMIZED-EMPTY:
+; OPTIMIZED-NEXT: <x1> vector loop: {
+; OPTIMIZED-NEXT: vector.body:
+; OPTIMIZED-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; OPTIMIZED-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
+; OPTIMIZED-NEXT: CLONE ir<%st.addr> = getelementptr inbounds nuw ir<%array>, vp<[[VP4]]>
+; OPTIMIZED-NEXT: CLONE ir<%ee.addr> = getelementptr inbounds nuw ir<%pred>, vp<[[VP4]]>
+; OPTIMIZED-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw ir<%ee.addr>
+; OPTIMIZED-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
+; OPTIMIZED-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
+; OPTIMIZED-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond>
+; OPTIMIZED-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>, ir<1>
+; OPTIMIZED-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds nuw ir<%st.addr>
+; OPTIMIZED-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>
+; OPTIMIZED-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
+; OPTIMIZED-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds nuw ir<%st.addr>
+; OPTIMIZED-NEXT: WIDEN store vp<[[VP8]]>, ir<%inc>, vp<%uncountable.exit.mask>
+; OPTIMIZED-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; OPTIMIZED-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of ir<%ee.cond>
+; OPTIMIZED-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; OPTIMIZED-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = or vp<[[VP9]]>, vp<[[VP10]]>
+; OPTIMIZED-NEXT: EMIT branch-on-cond vp<[[VP11]]>
+; OPTIMIZED-NEXT: No successors
+; OPTIMIZED-NEXT: }
+; OPTIMIZED-NEXT: Successor(s): middle.block
+; OPTIMIZED-EMPTY:
+; OPTIMIZED-NEXT: middle.block:
+; OPTIMIZED-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = add vp<[[VP3]]>, vp<[[VP6]]>
+; OPTIMIZED-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = icmp eq vp<[[VP13]]>, ir<20>
+; OPTIMIZED-NEXT: EMIT branch-on-cond vp<[[VP14]]>
+; OPTIMIZED-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; OPTIMIZED-EMPTY:
+; OPTIMIZED-NEXT: ir-bb<exit>:
+; OPTIMIZED-NEXT: No successors
+; OPTIMIZED-EMPTY:
+; OPTIMIZED-NEXT: scalar.ph:
+; OPTIMIZED-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP13]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; OPTIMIZED-NEXT: Successor(s): ir-bb<for.body>
+; OPTIMIZED-EMPTY:
+; OPTIMIZED-NEXT: ir-bb<for.body>:
+; OPTIMIZED-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; OPTIMIZED-NEXT: IR %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+; OPTIMIZED-NEXT: IR %data = load i16, ptr %st.addr, align 2
+; OPTIMIZED-NEXT: IR %inc = add nsw i16 %data, 1
+; OPTIMIZED-NEXT: IR store i16 %inc, ptr %st.addr, align 2
+; OPTIMIZED-NEXT: IR %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+; OPTIMIZED-NEXT: IR %ee.val = load i16, ptr %ee.addr, align 2
+; OPTIMIZED-NEXT: IR %ee.cond = icmp sgt i16 %ee.val, 500
+; OPTIMIZED-NEXT: No successors
+; OPTIMIZED-NEXT: }
+;
entry:
br label %for.body
@@ -28,6 +91,65 @@ exit:
}
define void @loop_contains_store_after_uncountable_exit(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; OPTIMIZED-LABEL: VPlan for loop in 'loop_contains_store_after_uncountable_exit'
+; OPTIMIZED: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; OPTIMIZED-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; OPTIMIZED-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; OPTIMIZED-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; OPTIMIZED-NEXT: Live-in ir<20> = original trip-count
+; OPTIMIZED-EMPTY:
+; OPTIMIZED-NEXT: ir-bb<entry>:
+; OPTIMIZED-NEXT: Successor(s): scalar.ph, vector.ph
+; OPTIMIZED-EMPTY:
+; OPTIMIZED-NEXT: vector.ph:
+; OPTIMIZED-NEXT: Successor(s): vector loop
+; OPTIMIZED-EMPTY:
+; OPTIMIZED-NEXT: <x1> vector loop: {
+; OPTIMIZED-NEXT: vector.body:
+; OPTIMIZED-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; OPTIMIZED-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
+; OPTIMIZED-NEXT: CLONE ir<%ee.addr> = getelementptr inbounds nuw ir<%pred>, vp<[[VP4]]>
+; OPTIMIZED-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw ir<%ee.addr>
+; OPTIMIZED-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
+; OPTIMIZED-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
+; OPTIMIZED-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond>
+; OPTIMIZED-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>, ir<1>
+; OPTIMIZED-NEXT: CLONE ir<%st.addr> = getelementptr ir<%array>, vp<[[VP4]]>
+; OPTIMIZED-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%st.addr>
+; OPTIMIZED-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>
+; OPTIMIZED-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
+; OPTIMIZED-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%st.addr>
+; OPTIMIZED-NEXT: WIDEN store vp<[[VP8]]>, ir<%inc>, vp<%uncountable.exit.mask>
+; OPTIMIZED-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; OPTIMIZED-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of ir<%ee.cond>
+; OPTIMIZED-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; OPTIMIZED-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = or vp<[[VP9]]>, vp<[[VP10]]>
+; OPTIMIZED-NEXT: EMIT branch-on-cond vp<[[VP11]]>
+; OPTIMIZED-NEXT: No successors
+; OPTIMIZED-NEXT: }
+; OPTIMIZED-NEXT: Successor(s): middle.block
+; OPTIMIZED-EMPTY:
+; OPTIMIZED-NEXT: middle.block:
+; OPTIMIZED-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = add vp<[[VP3]]>, vp<[[VP6]]>
+; OPTIMIZED-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = icmp eq vp<[[VP13]]>, ir<20>
+; OPTIMIZED-NEXT: EMIT branch-on-cond vp<[[VP14]]>
+; OPTIMIZED-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; OPTIMIZED-EMPTY:
+; OPTIMIZED-NEXT: ir-bb<exit>:
+; OPTIMIZED-NEXT: No successors
+; OPTIMIZED-EMPTY:
+; OPTIMIZED-NEXT: scalar.ph:
+; OPTIMIZED-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP13]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; OPTIMIZED-NEXT: Successor(s): ir-bb<for.body>
+; OPTIMIZED-EMPTY:
+; OPTIMIZED-NEXT: ir-bb<for.body>:
+; OPTIMIZED-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; OPTIMIZED-NEXT: IR %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+; OPTIMIZED-NEXT: IR %ee.val = load i16, ptr %ee.addr, align 2
+; OPTIMIZED-NEXT: IR %ee.cond = icmp sgt i16 %ee.val, 500
+; OPTIMIZED-NEXT: No successors
+; OPTIMIZED-NEXT: }
+;
entry:
br label %for.body
@@ -52,6 +174,72 @@ exit:
}
define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; OPTIMIZED-LABEL: VPlan for loop in 'uncountable_exit_with_live_out'
+; OPTIMIZED: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; OPTIMIZED-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; OPTIMIZED-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; OPTIMIZED-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; OPTIMIZED-NEXT: Live-in ir<20> = original trip-count
+; OPTIMIZED-EMPTY:
+; OPTIMIZED-NEXT: ir-bb<entry>:
+; OPTIMIZED-NEXT: Successor(s): scalar.ph, vector.ph
+; OPTIMIZED-EMPTY:
+; OPTIMIZED-NEXT: vector.ph:
+; OPTIMIZED-NEXT: Successor(s): vector loop
+; OPTIMIZED-EMPTY:
+; OPTIMIZED-NEXT: <x1> vector loop: {
+; OPTIMIZED-NEXT: vector.body:
+; OPTIMIZED-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; OPTIMIZED-NEXT: vp<[[VP4:%[0-9]+]]> = SCALAR-STEPS vp<[[VP3]]>, ir<1>, vp<[[VP0]]>
+; OPTIMIZED-NEXT: CLONE ir<%st.addr> = getelementptr inbounds nuw ir<%array>, vp<[[VP4]]>
+; OPTIMIZED-NEXT: CLONE ir<%ee.addr> = getelementptr inbounds nuw ir<%pred>, vp<[[VP4]]>
+; OPTIMIZED-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds nuw ir<%ee.addr>
+; OPTIMIZED-NEXT: WIDEN ir<%ee.val> = load vp<[[VP5]]>
+; OPTIMIZED-NEXT: WIDEN ir<%ee.cond> = icmp sgt ir<%ee.val>, ir<500>
+; OPTIMIZED-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = first-active-lane ir<%ee.cond>
+; OPTIMIZED-NEXT: EMIT vp<%uncountable.exit.mask> = active lane mask ir<0>, vp<[[VP6]]>, ir<1>
+; OPTIMIZED-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer inbounds nuw ir<%st.addr>
+; OPTIMIZED-NEXT: WIDEN ir<%data> = load vp<[[VP7]]>, vp<%uncountable.exit.mask>
+; OPTIMIZED-NEXT: WIDEN ir<%inc> = add nsw ir<%data>, ir<1>
+; OPTIMIZED-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer inbounds nuw ir<%st.addr>
+; OPTIMIZED-NEXT: WIDEN store vp<[[VP8]]>, ir<%inc>, vp<%uncountable.exit.mask>
+; OPTIMIZED-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; OPTIMIZED-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of ir<%ee.cond>
+; OPTIMIZED-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; OPTIMIZED-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = or vp<[[VP9]]>, vp<[[VP10]]>
+; OPTIMIZED-NEXT: EMIT branch-on-cond vp<[[VP11]]>
+; OPTIMIZED-NEXT: No successors
+; OPTIMIZED-NEXT: }
+; OPTIMIZED-NEXT: Successor(s): middle.block
+; OPTIMIZED-EMPTY:
+; OPTIMIZED-NEXT: middle.block:
+; OPTIMIZED-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-last-part ir<%data>
+; OPTIMIZED-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = extract-last-lane vp<[[VP13]]>
+; OPTIMIZED-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = add vp<[[VP3]]>, vp<[[VP6]]>
+; OPTIMIZED-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = icmp eq vp<[[VP15]]>, ir<20>
+; OPTIMIZED-NEXT: EMIT branch-on-cond vp<[[VP16]]>
+; OPTIMIZED-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; OPTIMIZED-EMPTY:
+; OPTIMIZED-NEXT: ir-bb<exit>:
+; OPTIMIZED-NEXT: IR %data.lcssa = phi i16 [ %data, %for.inc ], [ %data, %for.body ] (extra operand: vp<[[VP14]]> from middle.block)
+; OPTIMIZED-NEXT: No successors
+; OPTIMIZED-EMPTY:
+; OPTIMIZED-NEXT: scalar.ph:
+; OPTIMIZED-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP15]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; OPTIMIZED-NEXT: Successor(s): ir-bb<for.body>
+; OPTIMIZED-EMPTY:
+; OPTIMIZED-NEXT: ir-bb<for.body>:
+; OPTIMIZED-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; OPTIMIZED-NEXT: IR %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+; OPTIMIZED-NEXT: IR %data = load i16, ptr %st.addr, align 2
+; OPTIMIZED-NEXT: IR %inc = add nsw i16 %data, 1
+; OPTIMIZED-NEXT: IR store i16 %inc, ptr %st.addr, align 2
+; OPTIMIZED-NEXT: IR %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+; OPTIMIZED-NEXT: IR %ee.val = load i16, ptr %ee.addr, align 2
+; OPTIMIZED-NEXT: IR %ee.cond = icmp sgt i16 %ee.val, 500
+; OPTIMIZED-NEXT: No successors
+; OPTIMIZED-NEXT: }
+;
entry:
br label %for.body
@@ -74,5 +262,5 @@ for.inc:
exit:
ret i16 %data
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK: {{.*}}
+
+
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
index a46934f7b4a07..b89ea0b4c6ead 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
@@ -1,5 +1,5 @@
; REQUIRES: asserts
-; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -force-vector-width=4 -disable-output 2>&1 | FileCheck %s
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-early-exit-vectorization-with-side-effects -force-vector-width=4 -disable-output 2>&1 | FileCheck %s
;; This currently doesn't vectorize because the load used to determine the
;; uncountable exit condition has a second user (the store).
@@ -30,7 +30,7 @@ loop.end:
define void @loop_contains_store_condition_load_has_single_user(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_condition_load_has_single_user'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+; CHECK: LV: We can vectorize this loop!
entry:
br label %for.body
@@ -176,7 +176,7 @@ exit:
;; Alternatively, we could use masked.load.ff or vp.load.ff
define void @loop_contains_store_assumed_bounds(ptr noalias %array, ptr readonly %pred, i64 %n) {
; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_assumed_bounds'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+; CHECK: LV: We can vectorize this loop!
entry:
%n_bytes = mul nuw nsw i64 %n, 2
call void @llvm.assume(i1 true) [ "align"(ptr %pred, i64 2), "dereferenceable"(ptr %pred, i64 %n_bytes) ]
@@ -204,7 +204,7 @@ exit:
define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 dereferenceable(40) readonly %load.array, ptr align 2 noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_to_pointer_with_no_deref_info'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+; CHECK: LV: We can vectorize this loop!
entry:
br label %for.body
@@ -232,7 +232,7 @@ exit:
;; Vectorizeable, requires runtime checks and/or ff loads.
define void @loop_contains_store_unknown_bounds(ptr align 2 dereferenceable(100) noalias %array, ptr align 2 dereferenceable(100) readonly %pred, i64 %n) {
; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_unknown_bounds'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+; CHECK: LV: We can vectorize this loop!
entry:
br label %for.body
@@ -287,7 +287,7 @@ exit:
;; Vectorizeable, but we really want LICM to sink the store out of the loop
define void @loop_contains_store_to_invariant_location(ptr dereferenceable(40) readonly %array, ptr align 2 dereferenceable(40) readonly %pred, ptr noalias %store_addr) {
; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_to_invariant_location'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+; CHECK: LV: Not vectorizing: Cannot vectorize early exit loops with stores to loop-invariant addresses.
entry:
br label %for.body
@@ -313,7 +313,7 @@ exit:
define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_in_latch_block'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+; CHECK: LV: We can vectorize this loop!
entry:
br label %for.body
@@ -366,7 +366,7 @@ exit:
define void @loop_contains_store_decrementing_iv(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_decrementing_iv'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+; CHECK: LV: We can vectorize this loop!
entry:
br label %for.body
@@ -665,7 +665,7 @@ exit:
define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: LV: Checking a loop in 'uncountable_exit_with_live_out'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+; CHECK: LV: We can vectorize this loop!
entry:
br label %for.body
@@ -692,7 +692,8 @@ exit:
; Vectorizeable, requires improvements in dereferenceability checks
define void @uncountable_exit_with_constant_nonunit_stride(ptr dereferenceable(4000) noalias %array, ptr align 2 dereferenceable(4000) readonly %pred) {
; CHECK-LABEL: LV: Checking a loop in 'uncountable_exit_with_constant_nonunit_stride'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+; CHECK LV: We can vectorize this loop!
+; CHECK: LV: Not vectorizing: unable to calculate the loop count due to complex control flow.
entry:
br label %for.body
@@ -745,7 +746,7 @@ exit:
define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: LV: Checking a loop in 'uncountable_exit_with_separate_exit_block'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+; CHECK: LV: We can vectorize this loop!
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
index 173fb59a12402..6a1e2b0701186 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 | FileCheck %s
+; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 -force-target-supports-masked-memory-ops -enable-early-exit-vectorization-with-side-effects | FileCheck %s
;; See early_exit_store_legality.ll for reasons why a particular loop doesn't
;; vectorize yet.
@@ -50,22 +50,47 @@ loop.end:
define void @loop_contains_store_condition_load_has_single_user(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: define void @loop_contains_store_condition_load_has_single_user(
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) {
-; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
-; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ 0, %[[FOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP3]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
+; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP14]])
+; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
+; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP31]], ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP3]], 4
+; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP13]]
+; CHECK-NEXT: [[TMP41:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP40]])
+; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
+; CHECK-NEXT: [[TMP43:%.*]] = or i1 [[TMP41]], [[TMP42]]
+; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP44:%.*]] = add i64 [[TMP3]], [[TMP14]]
+; CHECK-NEXT: [[TMP45:%.*]] = icmp eq i64 [[TMP44]], 20
+; CHECK-NEXT: br i1 [[TMP45]], label %[[EXIT:.*]], label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[TMP44]], %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[DATA1:%.*]] = load i16, ptr [[ST_ADDR1]], align 2
+; CHECK-NEXT: [[INC1:%.*]] = add nsw i16 [[DATA1]], 1
+; CHECK-NEXT: store i16 [[INC1]], ptr [[ST_ADDR1]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -251,21 +276,50 @@ define void @loop_contains_store_assumed_bounds(ptr noalias %array, ptr readonly
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[N_BYTES:%.*]] = mul nuw nsw i64 [[N]], 2
; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[PRED]], i64 2), "dereferenceable"(ptr [[PRED]], i64 [[N_BYTES]]) ]
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
-; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[TMP2:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP11]], align 2
+; CHECK-NEXT: [[TMP12:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP12]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP13]])
+; CHECK-NEXT: [[TMP29:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP6]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
+; CHECK-NEXT: [[TMP30:%.*]] = add nsw <4 x i16> [[TMP29]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP30]], ptr align 2 [[TMP6]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 4
+; CHECK-NEXT: [[TMP39:%.*]] = freeze <4 x i1> [[TMP12]]
+; CHECK-NEXT: [[TMP40:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP39]])
+; CHECK-NEXT: [[TMP41:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP42:%.*]] = or i1 [[TMP40]], [[TMP41]]
+; CHECK-NEXT: br i1 [[TMP42]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP43:%.*]] = add i64 [[TMP2]], [[TMP13]]
+; CHECK-NEXT: [[TMP44:%.*]] = icmp eq i64 [[TMP43]], [[N]]
+; CHECK-NEXT: br i1 [[TMP44]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP43]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[DATA1:%.*]] = load i16, ptr [[ST_ADDR1]], align 2
+; CHECK-NEXT: [[INC1:%.*]] = add nsw i16 [[DATA1]], 1
+; CHECK-NEXT: store i16 [[INC1]], ptr [[ST_ADDR1]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -297,23 +351,49 @@ exit:
define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 dereferenceable(40) readonly %load.array, ptr align 2 noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: define void @loop_contains_store_to_pointer_with_no_deref_info(
; CHECK-SAME: ptr readonly align 2 dereferenceable(40) [[LOAD_ARRAY:%.*]], ptr noalias align 2 [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) {
-; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[LD_ADDR]], align 2
-; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[FOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds nuw i16, ptr [[LOAD_ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP4]], align 2
+; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP6]])
+; CHECK-NEXT: [[TMP26:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP0]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
+; CHECK-NEXT: [[TMP27:%.*]] = add nsw <4 x i16> [[TMP26]], splat (i16 1)
+; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP27]], ptr align 2 [[TMP29]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP41:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP40]])
+; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
+; CHECK-NEXT: [[TMP43:%.*]] = or i1 [[TMP41]], [[TMP42]]
+; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP44:%.*]] = add i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT: [[TMP45:%.*]] = icmp eq i64 [[TMP44]], 20
+; CHECK-NEXT: br i1 [[TMP45]], label %[[EXIT:.*]], label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[TMP44]], %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[LD_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[LOAD_ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[DATA1:%.*]] = load i16, ptr [[LD_ADDR1]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA1]], 1
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -345,21 +425,50 @@ define void @loop_contains_store_unknown_bounds(ptr align 2 dereferenceable(100)
; CHECK-LABEL: define void @loop_contains_store_unknown_bounds(
; CHECK-SAME: ptr noalias align 2 dereferenceable(100) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(100) [[PRED:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
-; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[TMP2:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP11]], align 2
+; CHECK-NEXT: [[TMP12:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP12]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP13]])
+; CHECK-NEXT: [[TMP29:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP6]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
+; CHECK-NEXT: [[TMP30:%.*]] = add nsw <4 x i16> [[TMP29]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP30]], ptr align 2 [[TMP6]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 4
+; CHECK-NEXT: [[TMP39:%.*]] = freeze <4 x i1> [[TMP12]]
+; CHECK-NEXT: [[TMP40:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP39]])
+; CHECK-NEXT: [[TMP41:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP42:%.*]] = or i1 [[TMP40]], [[TMP41]]
+; CHECK-NEXT: br i1 [[TMP42]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP43:%.*]] = add i64 [[TMP2]], [[TMP13]]
+; CHECK-NEXT: [[TMP44:%.*]] = icmp eq i64 [[TMP43]], [[N]]
+; CHECK-NEXT: br i1 [[TMP44]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP43]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[DATA1:%.*]] = load i16, ptr [[ST_ADDR1]], align 2
+; CHECK-NEXT: [[INC1:%.*]] = add nsw i16 [[DATA1]], 1
+; CHECK-NEXT: store i16 [[INC1]], ptr [[ST_ADDR1]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -389,22 +498,47 @@ exit:
define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: define void @loop_contains_store_in_latch_block(
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) {
-; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: [[SCALAR_PH:.*:]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
-; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[FOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP6]])
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP3]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
+; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP31]], ptr align 2 [[TMP3]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP41:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP40]])
+; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
+; CHECK-NEXT: [[TMP43:%.*]] = or i1 [[TMP41]], [[TMP42]]
+; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP44:%.*]] = add i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT: [[TMP45:%.*]] = icmp eq i64 [[TMP44]], 20
+; CHECK-NEXT: br i1 [[TMP45]], label %[[EXIT:.*]], label %[[SCALAR_PH1:.*]]
+; CHECK: [[SCALAR_PH1]]:
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[TMP44]], %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[EE_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR1]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -940,24 +1074,50 @@ exit:
define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: define i16 @uncountable_exit_with_live_out(
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
-; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ 0, %[[FOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP3]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
+; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP14]])
+; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
+; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP31]], ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP3]], 4
+; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP13]]
+; CHECK-NEXT: [[TMP41:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP40]])
+; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
+; CHECK-NEXT: [[TMP43:%.*]] = or i1 [[TMP41]], [[TMP42]]
+; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP44:%.*]] = extractelement <4 x i16> [[TMP30]], i32 3
+; CHECK-NEXT: [[TMP45:%.*]] = add i64 [[TMP3]], [[TMP14]]
+; CHECK-NEXT: [[TMP46:%.*]] = icmp eq i64 [[TMP45]], 20
+; CHECK-NEXT: br i1 [[TMP46]], label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[TMP45]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[DATA1:%.*]] = load i16, ptr [[ST_ADDR1]], align 2
+; CHECK-NEXT: [[INC1:%.*]] = add nsw i16 [[DATA1]], 1
+; CHECK-NEXT: store i16 [[INC1]], ptr [[ST_ADDR1]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[DATA_LCSSA:%.*]] = phi i16 [ [[DATA]], %[[FOR_INC]] ], [ [[DATA]], %[[FOR_BODY]] ]
+; CHECK-NEXT: [[DATA_LCSSA:%.*]] = phi i16 [ [[DATA1]], %[[FOR_INC]] ], [ [[DATA1]], %[[FOR_BODY1]] ], [ [[TMP44]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i16 [[DATA_LCSSA]]
;
entry:
@@ -1076,22 +1236,47 @@ exit:
define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: define i32 @uncountable_exit_with_separate_exit_block(
; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
-; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
-; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ 0, %[[FOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP3]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP12]], align 2
+; CHECK-NEXT: [[TMP13:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP13]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP14]])
+; CHECK-NEXT: [[TMP30:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
+; CHECK-NEXT: [[TMP31:%.*]] = add nsw <4 x i16> [[TMP30]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP31]], ptr align 2 [[TMP7]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP3]], 4
+; CHECK-NEXT: [[TMP40:%.*]] = freeze <4 x i1> [[TMP13]]
+; CHECK-NEXT: [[TMP41:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP40]])
+; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
+; CHECK-NEXT: [[TMP43:%.*]] = or i1 [[TMP41]], [[TMP42]]
+; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP44:%.*]] = add i64 [[TMP3]], [[TMP14]]
+; CHECK-NEXT: [[TMP45:%.*]] = icmp eq i64 [[TMP44]], 20
+; CHECK-NEXT: br i1 [[TMP45]], label %[[EXIT_COUNTABLE:.*]], label %[[SCALAR_PH:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
+; CHECK: [[FOR_BODY1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[TMP44]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
+; CHECK-NEXT: [[DATA1:%.*]] = load i16, ptr [[ST_ADDR1]], align 2
+; CHECK-NEXT: [[INC1:%.*]] = add nsw i16 [[DATA1]], 1
+; CHECK-NEXT: store i16 [[INC1]], ptr [[ST_ADDR1]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT_UNCOUNTABLE:.*]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT_COUNTABLE:.*]], label %[[FOR_BODY]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT_COUNTABLE]], label %[[FOR_BODY1]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: [[EXIT_COUNTABLE]]:
; CHECK-NEXT: ret i32 0
; CHECK: [[EXIT_UNCOUNTABLE]]:
>From a5896c26230316dc58ad4fbac39acbec73a00bd7 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Wed, 8 Apr 2026 14:33:12 +0000
Subject: [PATCH 2/3] Remove unnecessary checks
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 10 ++--------
1 file changed, 2 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 559949c03286d..cf61d2457fb99 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2769,9 +2769,8 @@ bool LoopVectorizationCostModel::isPredicatedInst(Instruction *I) const {
if (Legal->blockNeedsPredication(I->getParent()))
return true;
- // If we're not folding the tail by masking or vectorizing a loop with
- // uncountable exits and side effects, predication is unnecessary.
- if (!foldTailByMasking() && !Legal->hasUncountableExitWithSideEffects())
+ // If we're not folding the tail by masking, predication is unnecessary.
+ if (!foldTailByMasking())
return false;
// All that remain are instructions with side-effects originally executed in
@@ -4341,11 +4340,6 @@ bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
if (OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch())
return false;
- // Loops with uncountable exits and side effects (e.g. stores) are not yet
- // supported for epilogue vectorization.
- if (Legal->hasUncountableExitWithSideEffects())
- return false;
-
return true;
}
>From 1a6c4f156cea022a247ef16bf9161255e1a4c381 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Wed, 8 Apr 2026 16:15:26 +0000
Subject: [PATCH 3/3] Reimplement deref check for ee-with-stores in vplan
---
.../Vectorize/VPlanConstruction.cpp | 9 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 26 ++++--
.../Transforms/Vectorize/VPlanTransforms.h | 8 +-
.../AArch64/early_exit_with_stores.ll | 92 +++----------------
.../RISCV/early_exit_with_stores.ll | 92 +++----------------
.../early_exit_store_legality.ll | 2 +
.../LoopVectorize/early_exit_with_stores.ll | 88 +++---------------
7 files changed, 68 insertions(+), 249 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 91039d1d9b52b..280555ababbc1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1000,17 +1000,16 @@ bool VPlanTransforms::handleEarlyExits(VPlan &Plan, UncountableExitStyle Style,
// here from handleUncountableEarlyExits, but we need to improve
// detection of recipes which may write to memory.
if (Style != UncountableExitStyle::NoUncountableExit) {
- // Legality is still performing the dereferenceability check for
- // uncountable exit loops with stores; only the loads that contribute to
- // the condition must be verified, since every other memory operation will
- // be masked according to the exit condition.
+ // Dereferenceability is checked separately for uncountable exit loops with
+ // stores, as only the loads contributing to the exit condition need to
+ // be checked.
if (Style == UncountableExitStyle::ReadOnly &&
!areAllLoadsDereferenceable(HeaderVPBB, MiddleVPBB, TheLoop, PSE, DT,
AC))
return false;
// TODO: Check target preference for style.
return handleUncountableEarlyExits(Plan, HeaderVPBB, LatchVPBB, MiddleVPBB,
- Style);
+ TheLoop, PSE, DT, AC, Style);
}
// Disconnect countable early exits from the loop, leaving it with a single
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 2aba522a773bc..2bd78bc9a069b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -30,6 +30,7 @@
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/InstSimplifyFolder.h"
+#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
@@ -4167,8 +4168,9 @@ struct EarlyExitInfo {
/// early exit is taken or not.
static bool handleUncountableExitsWithSideEffects(
VPlan &Plan, SmallVectorImpl<EarlyExitInfo> &Exits,
- VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB,
- VPBasicBlock *MiddleVPBB) {
+ VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB, VPBasicBlock *MiddleVPBB,
+ Loop *TheLoop, PredicatedScalarEvolution &PSE, DominatorTree &DT,
+ AssumptionCache *AC) {
if (Plan.hasScalarVFOnly())
return false;
@@ -4209,6 +4211,13 @@ static bool handleUncountableExitsWithSideEffects(
}
assert(CondLoad && "Couldn't find load\n");
+ SmallVector<const SCEVPredicate *, 4> Predicates;
+ VPSingleDefRecipe *Load = cast<VPSingleDefRecipe>(CondLoad);
+ LoadInst *L = cast<LoadInst>(Load->getUnderlyingInstr());
+ if (!isDereferenceableAndAlignedInLoop(L, TheLoop, *PSE.getSE(), DT, AC,
+ &Predicates))
+ return false;
+
// Check GEPs to see if we can link them to a widen IV recipe with a step of
// 1; we're only interested in contiguous accesses for the condition load
// right now.
@@ -4327,11 +4336,10 @@ static bool handleUncountableExitsWithSideEffects(
return true;
}
-bool VPlanTransforms::handleUncountableEarlyExits(VPlan &Plan,
- VPBasicBlock *HeaderVPBB,
- VPBasicBlock *LatchVPBB,
- VPBasicBlock *MiddleVPBB,
- UncountableExitStyle Style) {
+bool VPlanTransforms::handleUncountableEarlyExits(
+ VPlan &Plan, VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB,
+ VPBasicBlock *MiddleVPBB, Loop *TheLoop, PredicatedScalarEvolution &PSE,
+ DominatorTree &DT, AssumptionCache *AC, UncountableExitStyle Style) {
VPDominatorTree VPDT(Plan);
VPBuilder Builder(LatchVPBB->getTerminator());
SmallVector<EarlyExitInfo> Exits;
@@ -4417,8 +4425,8 @@ bool VPlanTransforms::handleUncountableEarlyExits(VPlan &Plan,
VPValue *Combined =
Builder.createOr(IsAnyExitTaken, IsLatchExitTaken, LatchDL);
Builder.createNaryOp(VPInstruction::BranchOnCond, {Combined}, LatchDL);
- return handleUncountableExitsWithSideEffects(Plan, Exits, HeaderVPBB,
- LatchVPBB, MiddleVPBB);
+ return handleUncountableExitsWithSideEffects(
+ Plan, Exits, HeaderVPBB, LatchVPBB, MiddleVPBB, TheLoop, PSE, DT, AC);
}
// Create the vector.early.exit blocks.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index e3841b9d3ac80..ba101cbbae37b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -343,10 +343,10 @@ struct VPlanTransforms {
/// appropriate branching logic in the latch that handles early exits and the
/// latch exit condition. Multiple exits are handled with a dispatch block
/// that determines which exit to take based on lane-by-lane semantics.
- static bool handleUncountableEarlyExits(VPlan &Plan, VPBasicBlock *HeaderVPBB,
- VPBasicBlock *LatchVPBB,
- VPBasicBlock *MiddleVPBB,
- UncountableExitStyle Style);
+ static bool handleUncountableEarlyExits(
+ VPlan &Plan, VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB,
+ VPBasicBlock *MiddleVPBB, Loop *TheLoop, PredicatedScalarEvolution &PSE,
+ DominatorTree &DT, AssumptionCache *AC, UncountableExitStyle Style);
/// Replaces the exit condition from
/// (branch-on-cond eq CanonicalIVInc, VectorTripCount)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
index 5f5d338925be9..f59ee1fc430b8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
@@ -286,42 +286,9 @@ define void @loop_contains_store_assumed_bounds(ptr noalias %array, ptr readonly
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[N_BYTES:%.*]] = mul nuw nsw i64 [[N]], 2
; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[PRED]], i64 2), "dereferenceable"(ptr [[PRED]], i64 [[N_BYTES]]) ]
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
-; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
-; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
-; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP3]]
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[TMP7]]
-; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], [[N]]
-; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
; CHECK: [[FOR_BODY1]]:
-; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR1]], align 2
; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
@@ -329,11 +296,11 @@ define void @loop_contains_store_assumed_bounds(ptr noalias %array, ptr readonly
; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -393,7 +360,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP10]])
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
-; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[IV]], [[TMP7]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[TMP14]], 20
@@ -415,7 +382,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -446,43 +413,10 @@ exit:
define void @loop_contains_store_unknown_bounds(ptr align 2 dereferenceable(100) noalias %array, ptr align 2 dereferenceable(100) readonly %pred, i64 %n) {
; CHECK-LABEL: define void @loop_contains_store_unknown_bounds(
; CHECK-SAME: ptr noalias align 2 dereferenceable(100) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(100) [[PRED:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
-; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
-; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
-; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP3]]
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[TMP7]]
-; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], [[N]]
-; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
; CHECK: [[FOR_BODY1]]:
-; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR1]], align 2
; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
@@ -490,11 +424,11 @@ define void @loop_contains_store_unknown_bounds(ptr align 2 dereferenceable(100)
; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -551,7 +485,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP6]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
@@ -572,7 +506,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -1225,7 +1159,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY1]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV1]], [[TMP7]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
@@ -1246,7 +1180,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT_COUNTABLE]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT_COUNTABLE]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[EXIT_COUNTABLE]]:
; CHECK-NEXT: ret i32 0
; CHECK: [[EXIT_UNCOUNTABLE]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
index 138d04bc03edd..1d37b944ec02c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/early_exit_with_stores.ll
@@ -284,42 +284,9 @@ define void @loop_contains_store_assumed_bounds(ptr noalias %array, ptr readonly
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[N_BYTES:%.*]] = mul nuw nsw i64 [[N]], 2
; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[PRED]], i64 2), "dereferenceable"(ptr [[PRED]], i64 [[N_BYTES]]) ]
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
-; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
-; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
-; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP3]]
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[TMP7]]
-; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], [[N]]
-; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
; CHECK: [[FOR_BODY1]]:
-; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR1]], align 2
; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
@@ -327,11 +294,11 @@ define void @loop_contains_store_assumed_bounds(ptr noalias %array, ptr readonly
; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -391,7 +358,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP10]])
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
-; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[IV]], [[TMP7]]
; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[TMP14]], 20
@@ -413,7 +380,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -444,43 +411,10 @@ exit:
define void @loop_contains_store_unknown_bounds(ptr align 2 dereferenceable(100) noalias %array, ptr align 2 dereferenceable(100) readonly %pred, i64 %n) {
; CHECK-LABEL: define void @loop_contains_store_unknown_bounds(
; CHECK-SAME: ptr noalias align 2 dereferenceable(100) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(100) [[PRED:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
-; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
-; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
-; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
-; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[ST_ADDR]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP3]]
-; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
-; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[TMP7]]
-; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], [[N]]
-; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
; CHECK: [[FOR_BODY1]]:
-; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR1]], align 2
; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
@@ -488,11 +422,11 @@ define void @loop_contains_store_unknown_bounds(ptr align 2 dereferenceable(100)
; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -594,7 +528,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[TMP6]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
@@ -615,7 +549,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -1268,7 +1202,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
-; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[TMP7]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
@@ -1289,7 +1223,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT_COUNTABLE]], label %[[FOR_BODY1]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT_COUNTABLE]], label %[[FOR_BODY1]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[EXIT_COUNTABLE]]:
; CHECK-NEXT: ret i32 0
; CHECK: [[EXIT_UNCOUNTABLE]]:
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
index b89ea0b4c6ead..36d264651c279 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
@@ -177,6 +177,7 @@ exit:
define void @loop_contains_store_assumed_bounds(ptr noalias %array, ptr readonly %pred, i64 %n) {
; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_assumed_bounds'
; CHECK: LV: We can vectorize this loop!
+; CHECK: LV: No VPlans built.
entry:
%n_bytes = mul nuw nsw i64 %n, 2
call void @llvm.assume(i1 true) [ "align"(ptr %pred, i64 2), "dereferenceable"(ptr %pred, i64 %n_bytes) ]
@@ -233,6 +234,7 @@ exit:
define void @loop_contains_store_unknown_bounds(ptr align 2 dereferenceable(100) noalias %array, ptr align 2 dereferenceable(100) readonly %pred, i64 %n) {
; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_unknown_bounds'
; CHECK: LV: We can vectorize this loop!
+; CHECK: LV: No VPlans built.
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
index 6a1e2b0701186..70da01ca04c5e 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_with_stores.ll
@@ -276,38 +276,9 @@ define void @loop_contains_store_assumed_bounds(ptr noalias %array, ptr readonly
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[N_BYTES:%.*]] = mul nuw nsw i64 [[N]], 2
; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[PRED]], i64 2), "dereferenceable"(ptr [[PRED]], i64 [[N_BYTES]]) ]
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[TMP2:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP2]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP11]], align 2
-; CHECK-NEXT: [[TMP12:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP12]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP13]])
-; CHECK-NEXT: [[TMP29:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP6]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
-; CHECK-NEXT: [[TMP30:%.*]] = add nsw <4 x i16> [[TMP29]], splat (i16 1)
-; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP30]], ptr align 2 [[TMP6]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 4
-; CHECK-NEXT: [[TMP39:%.*]] = freeze <4 x i1> [[TMP12]]
-; CHECK-NEXT: [[TMP40:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP39]])
-; CHECK-NEXT: [[TMP41:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP42:%.*]] = or i1 [[TMP40]], [[TMP41]]
-; CHECK-NEXT: br i1 [[TMP42]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP43:%.*]] = add i64 [[TMP2]], [[TMP13]]
-; CHECK-NEXT: [[TMP44:%.*]] = icmp eq i64 [[TMP43]], [[N]]
-; CHECK-NEXT: br i1 [[TMP44]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP43]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
; CHECK: [[FOR_BODY1]]:
-; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[DATA1:%.*]] = load i16, ptr [[ST_ADDR1]], align 2
; CHECK-NEXT: [[INC1:%.*]] = add nsw i16 [[DATA1]], 1
@@ -315,11 +286,11 @@ define void @loop_contains_store_assumed_bounds(ptr noalias %array, ptr readonly
; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -372,7 +343,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK-NEXT: [[TMP41:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
; CHECK-NEXT: [[TMP43:%.*]] = or i1 [[TMP41]], [[TMP42]]
-; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP44:%.*]] = add i64 [[INDEX]], [[TMP6]]
; CHECK-NEXT: [[TMP45:%.*]] = icmp eq i64 [[TMP44]], 20
@@ -393,7 +364,7 @@ define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 deref
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -424,39 +395,10 @@ exit:
define void @loop_contains_store_unknown_bounds(ptr align 2 dereferenceable(100) noalias %array, ptr align 2 dereferenceable(100) readonly %pred, i64 %n) {
; CHECK-LABEL: define void @loop_contains_store_unknown_bounds(
; CHECK-SAME: ptr noalias align 2 dereferenceable(100) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(100) [[PRED:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[TMP2:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[TMP2]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP11]], align 2
-; CHECK-NEXT: [[TMP12:%.*]] = icmp sgt <4 x i16> [[WIDE_LOAD]], splat (i16 500)
-; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP12]], i1 false)
-; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[TMP13]])
-; CHECK-NEXT: [[TMP29:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr align 2 [[TMP6]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]], <4 x i16> poison)
-; CHECK-NEXT: [[TMP30:%.*]] = add nsw <4 x i16> [[TMP29]], splat (i16 1)
-; CHECK-NEXT: call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP30]], ptr align 2 [[TMP6]], <4 x i1> [[UNCOUNTABLE_EXIT_MASK]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 4
-; CHECK-NEXT: [[TMP39:%.*]] = freeze <4 x i1> [[TMP12]]
-; CHECK-NEXT: [[TMP40:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP39]])
-; CHECK-NEXT: [[TMP41:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: [[TMP42:%.*]] = or i1 [[TMP40]], [[TMP41]]
-; CHECK-NEXT: br i1 [[TMP42]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[TMP43:%.*]] = add i64 [[TMP2]], [[TMP13]]
-; CHECK-NEXT: [[TMP44:%.*]] = icmp eq i64 [[TMP43]], [[N]]
-; CHECK-NEXT: br i1 [[TMP44]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP43]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: br label %[[FOR_BODY1:.*]]
; CHECK: [[FOR_BODY1]]:
-; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
; CHECK-NEXT: [[ST_ADDR1:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV1]]
; CHECK-NEXT: [[DATA1:%.*]] = load i16, ptr [[ST_ADDR1]], align 2
; CHECK-NEXT: [[INC1:%.*]] = add nsw i16 [[DATA1]], 1
@@ -464,11 +406,11 @@ define void @loop_contains_store_unknown_bounds(ptr align 2 dereferenceable(100)
; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV1]]
; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
-; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -518,7 +460,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: [[TMP41:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
; CHECK-NEXT: [[TMP43:%.*]] = or i1 [[TMP41]], [[TMP42]]
-; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP44:%.*]] = add i64 [[INDEX]], [[TMP6]]
; CHECK-NEXT: [[TMP45:%.*]] = icmp eq i64 [[TMP44]], 20
@@ -538,7 +480,7 @@ define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias
; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -1094,7 +1036,7 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK-NEXT: [[TMP41:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
; CHECK-NEXT: [[TMP43:%.*]] = or i1 [[TMP41]], [[TMP42]]
-; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP44:%.*]] = extractelement <4 x i16> [[TMP30]], i32 3
; CHECK-NEXT: [[TMP45:%.*]] = add i64 [[TMP3]], [[TMP14]]
@@ -1115,7 +1057,7 @@ define i16 @uncountable_exit_with_live_out(ptr dereferenceable(40) noalias %arra
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY1]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[DATA_LCSSA:%.*]] = phi i16 [ [[DATA1]], %[[FOR_INC]] ], [ [[DATA1]], %[[FOR_BODY1]] ], [ [[TMP44]], %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i16 [[DATA_LCSSA]]
@@ -1256,7 +1198,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK-NEXT: [[TMP41:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP40]])
; CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20
; CHECK-NEXT: [[TMP43:%.*]] = or i1 [[TMP41]], [[TMP42]]
-; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP44:%.*]] = add i64 [[TMP3]], [[TMP14]]
; CHECK-NEXT: [[TMP45:%.*]] = icmp eq i64 [[TMP44]], 20
@@ -1276,7 +1218,7 @@ define i32 @uncountable_exit_with_separate_exit_block(ptr dereferenceable(40) no
; CHECK: [[FOR_INC]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
-; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT_COUNTABLE]], label %[[FOR_BODY1]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT_COUNTABLE]], label %[[FOR_BODY1]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[EXIT_COUNTABLE]]:
; CHECK-NEXT: ret i32 0
; CHECK: [[EXIT_UNCOUNTABLE]]:
More information about the llvm-commits
mailing list