[llvm] [LV] Vectorize early exit loops with stores using masking (PR #178454)
Graham Hunter via llvm-commits
llvm-commits at lists.llvm.org
Wed Feb 4 02:09:30 PST 2026
https://github.com/huntergr-arm updated https://github.com/llvm/llvm-project/pull/178454
>From 0baf518f95d26dc2e4cee8e24df05434d91f2e27 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Tue, 1 Jul 2025 13:08:48 +0000
Subject: [PATCH 1/2] Transform code
---
.../Vectorize/LoopVectorizationLegality.h | 6 +
.../Vectorize/LoopVectorizationLegality.cpp | 19 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 50 +-
.../Vectorize/VPlanConstruction.cpp | 6 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 190 ++-
.../Transforms/Vectorize/VPlanTransforms.h | 13 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 39 +-
.../AArch64/early_exit_with_stores.ll | 1260 +++++++++++++++++
.../early_exit_store_legality.ll | 18 +-
.../Transforms/Vectorize/VPlanTestBase.h | 5 +-
10 files changed, 1564 insertions(+), 42 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index f82fc588639dd..59cc98297c93c 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -483,6 +483,10 @@ class LoopVectorizationLegality {
return CountableExitingBlocks;
}
+ const ArrayRef<BasicBlock *> getUncountableExitingBlocks() const {
+ return UncountableExitingBlocksVec;
+ }
+
private:
/// Return true if the pre-header, exiting and latch blocks of \p Lp and all
/// its nested loops are considered legal for vectorization. These legal
@@ -738,6 +742,8 @@ class LoopVectorizationLegality {
/// the exact backedge taken count is not computable.
SmallVector<BasicBlock *, 4> CountableExitingBlocks;
+ SmallVector<BasicBlock *, 4> UncountableExitingBlocksVec;
+
/// True if the loop has uncountable early exits.
bool HasUncountableEarlyExit = false;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 1b000be130e49..978d91ea67338 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1874,6 +1874,7 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
<< *SymbolicMaxBTC << '\n');
HasUncountableEarlyExit = true;
UncountableExitWithSideEffects = HasSideEffects;
+ UncountableExitingBlocksVec.append(UncountableExitingBlocks);
return true;
}
@@ -2058,12 +2059,18 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
return false;
}
- // Bail out for state-changing loops with uncountable exits for now.
- if (UncountableExitWithSideEffects) {
- reportVectorizationFailure(
- "Writes to memory unsupported in early exit loops",
- "Cannot vectorize early exit loop with writes to memory",
- "WritesInEarlyExitLoop", ORE, TheLoop);
+ // TODO: Remove this restriction once we're sure it's safe to do so.
+ // If we're doing full masking we would need to store from the last
+ // active lane instead of the last lane according to VF. Doing the last
+ // iteration in the scalar loop should be ok, but we need to enforce it.
+ if (hasUncountableExitWithSideEffects() &&
+ !LAI->getStoresToInvariantAddresses().empty()) {
+ LLVM_DEBUG(dbgs() << "LV: Cannot vectorize early exit loops with stores to "
+ "loop-invariant addresses\n");
+ reportVectorizationFailure("Cannot vectorize EE loop with LI stores",
+ "Cannot vectorize early exit loops with stores "
+ "to loop-invariant addresses",
+ ORE, TheLoop);
return false;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 51895d1be27d7..a3ad2c34ca610 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -405,6 +405,12 @@ static cl::opt<bool> EnableEarlyExitVectorization(
cl::desc(
"Enable vectorization of early exit loops with uncountable exits."));
+static cl::opt<bool> EnableEarlyExitWithSideEffectVectorization(
+ "enable-early-exit-with-side-effect-vectorization", cl::init(false),
+ cl::Hidden,
+ cl::desc("Enable vectorization of early exit loops with uncountable exits "
+ "and side effects"));
+
static cl::opt<bool> ConsiderRegPressure(
"vectorizer-consider-reg-pressure", cl::init(false), cl::Hidden,
cl::desc("Discard VFs if their register pressure is too high."));
@@ -7309,6 +7315,7 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
});
assert(
(BestFactor.Width == LegacyVF.Width || BestPlan.hasEarlyExit() ||
+ Legal->hasUncountableExitWithSideEffects() ||
!Legal->getLAI()->getSymbolicStrides().empty() || UsesEVLGatherScatter ||
planContainsAdditionalSimplifications(
getPlanFor(BestFactor.Width), CostCtx, OrigLoop, BestFactor.Width) ||
@@ -8139,6 +8146,24 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
CM.getMaxSafeElements());
RUN_VPLAN_PASS(VPlanTransforms::optimizeEVLMasks, *Plan);
}
+
+ // If we have side effects (e.g. stores) in a loop with uncountable
+ // exits, we must ensure the early exit condition is known before any
+ // state changing operations take place.
+ if (!Plan->hasScalarVFOnly() &&
+ Legal->hasUncountableExitWithSideEffects()) {
+ ArrayRef<BasicBlock *> UncountableExits =
+ Legal->getUncountableExitingBlocks();
+ // TODO: Support more than one.
+ if (UncountableExits.size() > 1)
+ break;
+ auto *UncountableBR =
+ cast<BranchInst>(UncountableExits.front()->getTerminator());
+ if (!VPlanTransforms::runPass(
+ VPlanTransforms::handleUncountableExitsWithSideEffects, *Plan,
+ UncountableBR))
+ break;
+ }
assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
VPlans.push_back(std::move(Plan));
}
@@ -8163,7 +8188,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
return !CM.requiresScalarEpilogue(VF.isVector());
},
Range);
- VPlanTransforms::handleEarlyExits(*Plan, Legal->hasUncountableEarlyExit());
+ VPlanTransforms::handleEarlyExits(*Plan, Legal->hasUncountableEarlyExit(),
+ Legal->hasUncountableExitWithSideEffects());
VPlanTransforms::addMiddleCheck(*Plan, RequiresScalarEpilogueCheck,
CM.foldTailByMasking());
@@ -8423,8 +8449,10 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
MapVector<PHINode *, RecurrenceDescriptor>(),
SmallPtrSet<const PHINode *, 1>(), SmallPtrSet<PHINode *, 1>(),
/*AllowReordering=*/false);
- VPlanTransforms::handleEarlyExits(*Plan,
- /*HasUncountableExit*/ false);
+ VPlanTransforms::handleEarlyExits(
+ *Plan,
+ /*HasUncountableExit*/ false,
+ /*HasUncountableExitWithSideEffects=*/false);
VPlanTransforms::addMiddleCheck(*Plan, /*RequiresScalarEpilogue*/ true,
/*TailFolded*/ false);
@@ -9529,6 +9557,14 @@ bool LoopVectorizePass::processLoop(Loop *L) {
"UncountableEarlyExitLoopsDisabled", ORE, L);
return false;
}
+ if (LVL.hasUncountableExitWithSideEffects() &&
+ !EnableEarlyExitWithSideEffectVectorization) {
+ reportVectorizationFailure("Auto-vectorization of loops with uncountable "
+ "early exit and side effects is not enabled",
+ "UncountableEarlyExitSideEffectLoopsDisabled",
+ ORE, L);
+ return false;
+ }
SmallVector<BasicBlock *, 8> ExitingBlocks;
L->getExitingBlocks(ExitingBlocks);
// TODO: Support multiple uncountable early exits.
@@ -9796,6 +9832,14 @@ bool LoopVectorizePass::processLoop(Loop *L) {
IC = 1;
}
+ if (InterleaveLoop && LVL.hasUncountableExitWithSideEffects()) {
+ LLVM_DEBUG(dbgs() << "LV: Not interleaving due to EE with side effects.\n");
+ IntDiagMsg = {"EEWithSideEffectsPreventsInterleaving",
+ "Unable to interleave due to early exit with side effects."};
+ InterleaveLoop = false;
+ IC = 1;
+ }
+
// Emit diagnostic messages, if any.
const char *VAPassName = Hints.vectorizeAnalysisPassName();
if (!VectorizeLoop && !InterleaveLoop) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index ec4c0e0321e66..d7dc4d702dc2b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -873,7 +873,8 @@ void VPlanTransforms::createInLoopReductionRecipes(
}
void VPlanTransforms::handleEarlyExits(VPlan &Plan,
- bool HasUncountableEarlyExit) {
+ bool HasUncountableEarlyExit,
+ bool HasUncountableExitWithSideEffects) {
auto *MiddleVPBB = cast<VPBasicBlock>(
Plan.getScalarHeader()->getSinglePredecessor()->getPredecessors()[0]);
auto *LatchVPBB = cast<VPBasicBlock>(MiddleVPBB->getSinglePredecessor());
@@ -893,7 +894,8 @@ void VPlanTransforms::handleEarlyExits(VPlan &Plan,
assert(!HandledUncountableEarlyExit &&
"can handle exactly one uncountable early exit");
handleUncountableEarlyExit(cast<VPBasicBlock>(Pred), EB, Plan,
- cast<VPBasicBlock>(HeaderVPB), LatchVPBB);
+ cast<VPBasicBlock>(HeaderVPB), LatchVPBB,
+ HasUncountableExitWithSideEffects);
HandledUncountableEarlyExit = true;
} else {
for (VPRecipeBase &R : EB->phis())
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 7ffd42cdda614..4317006534384 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4029,13 +4029,11 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
R->eraseFromParent();
}
-void VPlanTransforms::handleUncountableEarlyExit(VPBasicBlock *EarlyExitingVPBB,
- VPBasicBlock *EarlyExitVPBB,
- VPlan &Plan,
- VPBasicBlock *HeaderVPBB,
- VPBasicBlock *LatchVPBB) {
+void VPlanTransforms::handleUncountableEarlyExit(
+ VPBasicBlock *EarlyExitingVPBB, VPBasicBlock *EarlyExitVPBB, VPlan &Plan,
+ VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB, bool HasSideEffects) {
auto *MiddleVPBB = cast<VPBasicBlock>(LatchVPBB->getSuccessors()[0]);
- if (!EarlyExitVPBB->getSinglePredecessor() &&
+ if (!HasSideEffects && !EarlyExitVPBB->getSinglePredecessor() &&
EarlyExitVPBB->getPredecessors()[1] == MiddleVPBB) {
assert(EarlyExitVPBB->getNumPredecessors() == 2 &&
EarlyExitVPBB->getPredecessors()[0] == EarlyExitingVPBB &&
@@ -4061,6 +4059,23 @@ void VPlanTransforms::handleUncountableEarlyExit(VPBasicBlock *EarlyExitingVPBB,
// [0] vector.early.exit, [1] middle block, [2] header (continue looping).
VPValue *IsEarlyExitTaken =
Builder.createNaryOp(VPInstruction::AnyOf, {CondToEarlyExit});
+
+ auto *LatchExitingBranch = cast<VPInstruction>(LatchVPBB->getTerminator());
+ assert(LatchExitingBranch->getOpcode() == VPInstruction::BranchOnCount &&
+ "Unexpected terminator");
+ auto *IsLatchExitTaken =
+ Builder.createICmp(CmpInst::ICMP_EQ, LatchExitingBranch->getOperand(0),
+ LatchExitingBranch->getOperand(1));
+ DebugLoc LatchDL = LatchExitingBranch->getDebugLoc();
+ LatchExitingBranch->eraseFromParent();
+
+ if (HasSideEffects) {
+ Builder.setInsertPoint(LatchVPBB);
+ VPInstruction *Combined = Builder.createNaryOp(
+ Instruction::Or, {IsEarlyExitTaken, IsLatchExitTaken}, LatchDL);
+ Builder.createNaryOp(VPInstruction::BranchOnCond, {Combined});
+ return;
+ }
VPBasicBlock *VectorEarlyExitVPBB =
Plan.createVPBasicBlock("vector.early.exit");
VectorEarlyExitVPBB->setParent(EarlyExitVPBB->getParent());
@@ -4098,15 +4113,6 @@ void VPlanTransforms::handleUncountableEarlyExit(VPBasicBlock *EarlyExitingVPBB,
// loop with a multi-conditional branch exiting to vector early exit if the
// early exit has been taken, exiting to middle block if the original
// condition of the vector latch is true, otherwise continuing back to header.
- auto *LatchExitingBranch = cast<VPInstruction>(LatchVPBB->getTerminator());
- assert(LatchExitingBranch->getOpcode() == VPInstruction::BranchOnCount &&
- "Unexpected terminator");
- auto *IsLatchExitTaken =
- Builder.createICmp(CmpInst::ICMP_EQ, LatchExitingBranch->getOperand(0),
- LatchExitingBranch->getOperand(1));
-
- DebugLoc LatchDL = LatchExitingBranch->getDebugLoc();
- LatchExitingBranch->eraseFromParent();
Builder.setInsertPoint(LatchVPBB);
Builder.createNaryOp(VPInstruction::BranchOnTwoConds,
@@ -4116,6 +4122,160 @@ void VPlanTransforms::handleUncountableEarlyExit(VPBasicBlock *EarlyExitingVPBB,
VectorEarlyExitVPBB->setPredecessors({LatchVPBB});
}
+bool VPlanTransforms::handleUncountableExitsWithSideEffects(
+ VPlan &Plan, BranchInst *UncountableBR) {
+ if (Plan.hasScalarVFOnly())
+ return false;
+
+ // We can abandon a vplan entirely if we return false here, so we shouldn't
+ // crash if some earlier assumptions on scalar IR don't hold for the vplan
+ // version of the loop.
+ VPCanonicalIVPHIRecipe *IV = Plan.getVectorLoopRegion()->getCanonicalIV();
+ VPInstruction *IVUpdate = dyn_cast<VPInstruction>(IV->getBackedgeValue());
+ if (!IVUpdate)
+ return false;
+
+ SmallVector<VPRecipeBase *, 2> GEPs;
+ SmallVector<VPRecipeBase *, 8> ConditionRecipes;
+
+ std::optional<VPValue *> Cond =
+ vputils::getRecipesForUncountableExit(Plan, ConditionRecipes, GEPs);
+ if (!Cond)
+ return false;
+
+ // Find load contributing to condition.
+ VPWidenLoadRecipe *CondLoad = nullptr;
+ for (auto *Recipe : ConditionRecipes) {
+ if (auto *L = dyn_cast<VPWidenLoadRecipe>(Recipe)) {
+ // TODO: Support more than one load. Needs legality updates too.
+ assert(CondLoad == nullptr && "Too many condition loads\n");
+ CondLoad = L;
+ }
+ }
+ assert(CondLoad && "Couldn't find load\n");
+
+ // Check GEPs to see if we can link them to the canonical IV.
+ using namespace llvm::VPlanPatternMatch;
+ for (auto *GEP : GEPs)
+ if (!match(GEP,
+ m_GetElementPtr(m_LiveIn(),
+ m_ScalarIVSteps(m_Specific(IV), m_SpecificInt(1),
+ m_Specific(&Plan.getVF())))))
+ return false;
+
+ // Do we need to move? Find the first memory operation that isn't the
+ // load for the early exit condition comparison.
+ VPRecipeBase *FirstMemOp = nullptr;
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_deep(Plan.getVectorLoopRegion()))) {
+ for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+ if (auto *MemOp = dyn_cast<VPWidenMemoryRecipe>(&R);
+ MemOp != nullptr && MemOp != CondLoad) {
+ FirstMemOp = &R;
+ break;
+ }
+ }
+
+ if (FirstMemOp)
+ break;
+ }
+
+ // TODO: Assert instead?
+ if (!FirstMemOp)
+ return false;
+
+ // If another memory operation would take place before the comparison to
+ // determine whether to exit early, move the comparison (and supporting
+ // recipes) before it.)
+ VPRecipeBase *CondR = (*Cond)->getDefiningRecipe();
+ VPDominatorTree VPDT(Plan);
+ if (!VPDT.properlyDominates(CondR, FirstMemOp)) {
+ for (auto *Recipe : reverse(ConditionRecipes))
+ Recipe->moveBefore(*FirstMemOp->getParent(), FirstMemOp->getIterator());
+
+ CondR->moveBefore(*FirstMemOp->getParent(), FirstMemOp->getIterator());
+ }
+
+ // Create a mask to represent all lanes that fully execute in the vector loop,
+ // stopping short of any early exit.
+ VPBuilder MaskBuilder(FirstMemOp);
+ VPValue *FirstActive =
+ MaskBuilder.createNaryOp(VPInstruction::FirstActiveLane, *Cond);
+ VPValue *ALMMultiplier =
+ Plan.getConstantInt(Plan.getVectorLoopRegion()->getCanonicalIVType(), 1);
+ VPValue *Zero =
+ Plan.getConstantInt(Plan.getVectorLoopRegion()->getCanonicalIVType(), 0);
+ VPValue *Mask = MaskBuilder.createNaryOp(VPInstruction::ActiveLaneMask,
+ {Zero, FirstActive, ALMMultiplier},
+ nullptr, "uncountable.exit.mask");
+
+ // Convert all other memory operations to use the mask.
+ // TODO: If a (non-condition) load is guaranteed to be dereferenceable, does
+ // it matter if we don't convert it?
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_deep(Plan.getVectorLoopRegion()))) {
+ for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+ if (auto *MemOp = dyn_cast<VPWidenMemoryRecipe>(&R)) {
+ if (MemOp == CondLoad)
+ continue;
+
+ if (auto *St = dyn_cast<VPWidenStoreRecipe>(MemOp)) {
+ auto *NewStore = new VPWidenStoreRecipe(
+ (cast<StoreInst>(St->getIngredient())), St->getAddr(),
+ St->getStoredValue(), Mask, St->isConsecutive(), St->isReverse(),
+ *St, St->getDebugLoc());
+ NewStore->insertBefore(St);
+ St->eraseFromParent();
+ } else if (auto *Ld = dyn_cast<VPWidenLoadRecipe>(MemOp)) {
+ auto *NewLoad = new VPWidenLoadRecipe(
+ *(cast<LoadInst>(Ld->getUnderlyingValue())), Ld->getOperand(0),
+ Mask, Ld->isConsecutive(), Ld->isReverse(), *Ld,
+ Ld->getDebugLoc());
+ NewLoad->insertBefore(Ld);
+ Ld->replaceAllUsesWith(NewLoad);
+ Ld->eraseFromParent();
+ } else {
+ llvm_unreachable("wrong memop?");
+ }
+ }
+ }
+ }
+
+ // Update middle block branch to compare (IV + however many lanes were active)
+ // against the full trip count, since we may be exiting the vector loop early.
+ // If we didn't take an early exit, we should get the equivalent of VF from
+ // the cttz.elts.
+ VPBasicBlock *MiddleBlock = Plan.getMiddleBlock();
+ VPRecipeBase *OldTerminator = MiddleBlock->getTerminator();
+ VPBuilder MiddleBuilder(OldTerminator);
+ // TODO: Create popcount of mask another way... zext + reduce_add?
+ // For SVE at least we want to be able to fold away the
+ // brkb+cntp+whilelo in the loop to just brkb. We need a cntp
+ // outside the loop instead... maybe copy in CodeGenPrepare?
+ VPTypeAnalysis TypeInfo(Plan);
+ VPValue *ExitIV =
+ MiddleBuilder.createNaryOp(Instruction::Add, {IV, FirstActive});
+ VPValue *FullTC =
+ MiddleBuilder.createICmp(CmpInst::ICMP_EQ, ExitIV, Plan.getTripCount());
+ OldTerminator->setOperand(0, FullTC);
+
+ // Update resume phi in scalar.ph...
+ VPBasicBlock *ScalarPH = Plan.getScalarPreheader();
+ auto Phis = ScalarPH->phis();
+ bool PhiFound = false;
+ for (auto &Phi : Phis) {
+ // TODO: Handle more than one Phi; re-derive from IV.
+ // TODO: Handle reductions, one day...
+ if (PhiFound)
+ return false;
+ PhiFound = true;
+ VPPhi *ContinueIV = cast<VPPhi>(&Phi);
+ ContinueIV->setOperand(0, ExitIV);
+ }
+
+ return true;
+}
+
/// This function tries convert extended in-loop reductions to
/// VPExpressionRecipe and clamp the \p Range if it is beneficial and
/// valid. The created recipe must be decomposed to its constituent
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 49d15f15ece94..63da7d299595c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -135,8 +135,9 @@ struct VPlanTransforms {
ElementCount MinVF);
/// Update \p Plan to account for all early exits.
- LLVM_ABI_FOR_TEST static void handleEarlyExits(VPlan &Plan,
- bool HasUncountableExit);
+ LLVM_ABI_FOR_TEST static void
+ handleEarlyExits(VPlan &Plan, bool HasUncountableExit,
+ bool HasUncountableExitWithSideEffects);
/// If a check is needed to guard executing the scalar epilogue loop, it will
/// be added to the middle block.
@@ -311,7 +312,13 @@ struct VPlanTransforms {
static void handleUncountableEarlyExit(VPBasicBlock *EarlyExitingVPBB,
VPBasicBlock *EarlyExitVPBB,
VPlan &Plan, VPBasicBlock *HeaderVPBB,
- VPBasicBlock *LatchVPBB);
+ VPBasicBlock *LatchVPBB,
+ bool HasSideEffects);
+
+ /// Update \p Plan to mask memory operations in the loop based on whether
+ /// the early exit is taken or not.
+ static bool handleUncountableExitsWithSideEffects(VPlan &Plan,
+ BranchInst *UncountableBR);
/// Replaces the exit condition from
/// (branch-on-cond eq CanonicalIVInc, VectorTripCount)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 9c013b27c17ab..d53a6950f37da 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -515,11 +515,16 @@ vputils::getRecipesForUncountableExit(VPlan &Plan,
// }
// Find the uncountable loop exit condition.
+ // TODO: Determine what we should be matching here after reviewers have
+ // had a chance to look; BranchOnTwoConds or BranchOnCond with Or.
auto *Region = Plan.getVectorLoopRegion();
VPValue *UncountableCondition = nullptr;
if (!match(Region->getExitingBasicBlock()->getTerminator(),
- m_BranchOnTwoConds(m_AnyOf(m_VPValue(UncountableCondition)),
- m_VPValue())))
+ m_CombineOr(
+ m_BranchOnTwoConds(m_AnyOf(m_VPValue(UncountableCondition)),
+ m_VPValue()),
+ m_BranchOnCond(m_c_BinaryOr(
+ m_AnyOf(m_VPValue(UncountableCondition)), m_VPValue())))))
return std::nullopt;
SmallVector<VPValue *, 4> Worklist;
@@ -549,17 +554,45 @@ vputils::getRecipesForUncountableExit(VPlan &Plan,
if (Load->isMasked())
return std::nullopt;
+ Recipes.push_back(Load);
+
+ // Look through vector-pointer recipes.
VPValue *GEP = Load->getAddr();
+ if (auto *VecPtrR = dyn_cast<VPVectorPointerRecipe>(GEP)) {
+ Recipes.push_back(VecPtrR);
+ GEP = VecPtrR->getOperand(0);
+ }
+
+ // We only support two-operand GEPS with match
+ if (auto *R = GEP->getDefiningRecipe(); !R || R->getNumOperands() != 2)
+ return std::nullopt;
+
if (!match(GEP, m_GetElementPtr(m_LiveIn(), m_VPValue())))
return std::nullopt;
- Recipes.push_back(Load);
Recipes.push_back(GEP->getDefiningRecipe());
GEPs.push_back(GEP->getDefiningRecipe());
} else
return std::nullopt;
}
+ // If we couldn't match anything, don't return the condition. It may be
+ // defined outside the loop.
+ if (Recipes.empty())
+ return std::nullopt;
+
+#ifndef NDEBUG
+ // Check dominance ordering
+ VPRecipeBase *RA = Recipes.front();
+ VPDominatorTree VPDT(Plan);
+ bool Ordered = all_of(drop_begin(Recipes), [&VPDT, &RA](VPRecipeBase *RB) {
+ bool Dominates = VPDT.properlyDominates(RB, RA);
+ RA = RB;
+ return Dominates;
+ });
+ assert(Ordered && "Uncountable exit recipes unordered");
+#endif
+
return UncountableCondition;
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
new file mode 100644
index 0000000000000..79569ce923745
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
@@ -0,0 +1,1260 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S < %s -p loop-vectorize -enable-early-exit-with-side-effect-vectorization -mattr=+sve2 2>&1 | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define i64 @loop_contains_store(ptr %dest) {
+; CHECK-LABEL: define i64 @loop_contains_store(
+; CHECK-SAME: ptr [[DEST:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 4
+; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], %[[LOOP_INC:.*]] ], [ 3, %[[ENTRY]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[INDEX]]
+; CHECK-NEXT: [[LD1:%.*]] = load i32, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[DEST]], i64 [[INDEX]]
+; CHECK-NEXT: store i32 [[LD1]], ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[LD1]], 1
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP_INC]], label %[[LOOP_END:.*]]
+; CHECK: [[LOOP_INC]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP]], label %[[LOOP_END]]
+; CHECK: [[LOOP_END]]:
+; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], %[[LOOP]] ], [ 67, %[[LOOP_INC]] ]
+; CHECK-NEXT: ret i64 [[RETVAL]]
+;
+entry:
+ %p1 = alloca [1024 x i8]
+ call void @init_mem(ptr %p1, i64 1024)
+ br label %loop
+
+loop:
+ %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
+ %arrayidx = getelementptr inbounds i32, ptr %p1, i64 %index
+ %ld1 = load i32, ptr %arrayidx, align 1
+ %arrayidx2 = getelementptr inbounds i32, ptr %dest, i64 %index
+ store i32 %ld1, ptr %arrayidx2, align 4
+ %cmp = icmp eq i32 %ld1, 1
+ br i1 %cmp, label %loop.inc, label %loop.end
+
+loop.inc:
+ %index.next = add i64 %index, 1
+ %exitcond = icmp ne i64 %index.next, 67
+ br i1 %exitcond, label %loop, label %loop.end
+
+loop.end:
+ %retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
+ ret i64 %retval
+}
+
+define void @loop_contains_store_condition_load_has_single_user(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: define void @loop_contains_store_condition_load_has_single_user(
+; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[TMP4]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[TMP4]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[INDEX]], [[TMP7]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
+; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_ee_condition_is_invariant(ptr dereferenceable(40) noalias %array, i16 %ee.val) {
+; CHECK-LABEL: define void @loop_contains_store_ee_condition_is_invariant(
+; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], i16 [[EE_VAL:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: define void @loop_contains_store_fcmp_condition(
+; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw half, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load half, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = fcmp ugt half [[EE_VAL]], 0xH5FD0
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw half, ptr %pred, i64 %iv
+ %ee.val = load half, ptr %ee.addr, align 2
+ %ee.cond = fcmp ugt half %ee.val, 500.0
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_safe_dependency(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(96) %pred) {
+; CHECK-LABEL: define void @loop_contains_store_safe_dependency(
+; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr align 2 dereferenceable(96) [[PRED:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[PRED_PLUS_8:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 8
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED_PLUS_8]], i64 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: [[SOME_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: store i16 42, ptr [[SOME_ADDR]], align 2
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %pred.plus.8 = getelementptr inbounds nuw i16, ptr %pred, i64 8
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred.plus.8, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ %some.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ store i16 42, ptr %some.addr, align 2
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_unsafe_dependency(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(80) readonly %pred) {
+; CHECK-LABEL: define void @loop_contains_store_unsafe_dependency(
+; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(80) [[PRED:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[UNKNOWN_OFFSET:%.*]] = call i64 @get_an_unknown_offset()
+; CHECK-NEXT: [[UNKNOWN_CMP:%.*]] = icmp ult i64 [[UNKNOWN_OFFSET]], 20
+; CHECK-NEXT: [[CLAMPED_OFFSET:%.*]] = select i1 [[UNKNOWN_CMP]], i64 [[UNKNOWN_OFFSET]], i64 20
+; CHECK-NEXT: [[UNKNOWN_BASE:%.*]] = getelementptr i16, ptr [[PRED]], i64 [[CLAMPED_OFFSET]]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[UNKNOWN_BASE]], i64 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: [[SOME_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: store i16 42, ptr [[SOME_ADDR]], align 2
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %unknown.offset = call i64 @get_an_unknown_offset()
+ %unknown.cmp = icmp ult i64 %unknown.offset, 20
+ %clamped.offset = select i1 %unknown.cmp, i64 %unknown.offset, i64 20
+ %unknown.base = getelementptr i16, ptr %pred, i64 %clamped.offset
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %unknown.base, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ %some.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ store i16 42, ptr %some.addr, align 2
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_assumed_bounds(ptr noalias %array, ptr readonly %pred, i32 %n) {
+; CHECK-LABEL: define void @loop_contains_store_assumed_bounds(
+; CHECK-SAME: ptr noalias [[ARRAY:%.*]], ptr readonly [[PRED:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[N_BYTES:%.*]] = mul nuw nsw i32 [[N]], 2
+; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[PRED]], i64 2), "dereferenceable"(ptr [[PRED]], i32 [[N_BYTES]]) ]
+; CHECK-NEXT: [[TC:%.*]] = sext i32 [[N]] to i64
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[TC]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %n_bytes = mul nuw nsw i32 %n, 2
+ call void @llvm.assume(i1 true) [ "align"(ptr %pred, i64 2), "dereferenceable"(ptr %pred, i32 %n_bytes) ]
+ %tc = sext i32 %n to i64
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, %tc
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 dereferenceable(40) readonly %load.array, ptr align 2 noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: define void @loop_contains_store_to_pointer_with_no_deref_info(
+; CHECK-SAME: ptr readonly align 2 dereferenceable(40) [[LOAD_ARRAY:%.*]], ptr noalias align 2 [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i16, ptr [[LOAD_ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP5]], align 2
+; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[TMP4]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[TMP9]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[TMP10:%.*]] = freeze <vscale x 8 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP10]])
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP13:%.*]] = or i1 [[TMP11]], [[TMP12]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[INDEX]], [[TMP7]]
+; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[TMP14]], 20
+; CHECK-NEXT: br i1 [[TMP15]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[LD_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[LOAD_ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[LD_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %ld.addr = getelementptr inbounds nuw i16, ptr %load.array, i64 %iv
+ %data = load i16, ptr %ld.addr, align 2
+ %inc = add nsw i16 %data, 1
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_unknown_bounds(ptr align 2 dereferenceable(100) noalias %array, ptr align 2 dereferenceable(100) readonly %pred, i64 %n) {
+; CHECK-LABEL: define void @loop_contains_store_unknown_bounds(
+; CHECK-SAME: ptr noalias align 2 dereferenceable(100) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(100) [[PRED:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, %n
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_volatile(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: define void @loop_contains_store_volatile(
+; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store volatile i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store volatile i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_to_invariant_location(ptr dereferenceable(40) readonly %array, ptr align 2 dereferenceable(40) readonly %pred, ptr noalias %store_addr) {
+; CHECK-LABEL: define void @loop_contains_store_to_invariant_location(
+; CHECK-SAME: ptr readonly dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]], ptr noalias [[STORE_ADDR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[STORE_ADDR]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %store_addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: define void @loop_contains_store_in_latch_block(
+; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 20, [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 20, [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 20, [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP4]], align 2
+; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt <vscale x 8 x i16> [[WIDE_LOAD]], splat (i16 500)
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i16, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv8i1(<vscale x 8 x i1> [[TMP5]], i1 false)
+; CHECK-NEXT: [[UNCOUNTABLE_EXIT_MASK:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[TMP7]])
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.masked.load.nxv8i16.p0(ptr align 2 [[TMP6]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]], <vscale x 8 x i16> poison)
+; CHECK-NEXT: [[TMP8:%.*]] = add nsw <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], splat (i16 1)
+; CHECK-NEXT: call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> [[TMP8]], ptr align 2 [[TMP6]], <vscale x 8 x i1> [[UNCOUNTABLE_EXIT_MASK]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <vscale x 8 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv8i1(<vscale x 8 x i1> [[TMP9]])
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP10]], [[TMP11]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[INDEX]], [[TMP7]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 20
+; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT]], label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_requiring_alias_check(ptr dereferenceable(40) %array, ptr align 2 dereferenceable(40) %pred) {
+; CHECK-LABEL: define void @loop_contains_store_requiring_alias_check(
+; CHECK-SAME: ptr dereferenceable(40) [[ARRAY:%.*]], ptr align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_condition_load_is_chained(ptr dereferenceable(40) noalias %array, ptr align 8 dereferenceable(160) readonly %offsets, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: define void @loop_contains_store_condition_load_is_chained(
+; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 8 dereferenceable(160) [[OFFSETS:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[GATHER_ADDR:%.*]] = getelementptr inbounds nuw i64, ptr [[OFFSETS]], i64 [[IV]]
+; CHECK-NEXT: [[EE_OFFSET:%.*]] = load i64, ptr [[GATHER_ADDR]], align 8
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[EE_OFFSET]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %gather.addr = getelementptr inbounds nuw i64, ptr %offsets, i64 %iv
+ %ee.offset = load i64, ptr %gather.addr, align 8
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %ee.offset
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_decrementing_iv(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: define void @loop_contains_store_decrementing_iv(
+; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 19, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = sub nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 0
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 19, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = sub nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 0
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_condition_load_requires_gather(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(512) readonly %pred, ptr align 1 dereferenceable(20) readonly %offsets) {
+; CHECK-LABEL: define void @loop_contains_store_condition_load_requires_gather(
+; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(512) [[PRED:%.*]], ptr readonly align 1 dereferenceable(20) [[OFFSETS:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[OFFSET_ADDR:%.*]] = getelementptr inbounds nuw i8, ptr [[OFFSETS]], i64 [[IV]]
+; CHECK-NEXT: [[OFFSET:%.*]] = load i8, ptr [[OFFSET_ADDR]], align 1
+; CHECK-NEXT: [[OFFSET_ZEXT:%.*]] = zext i8 [[OFFSET]] to i64
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[OFFSET_ZEXT]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %offset.addr = getelementptr inbounds nuw i8, ptr %offsets, i64 %iv
+ %offset = load i8, ptr %offset.addr, align 1
+ %offset.zext = zext i8 %offset to i64
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %offset.zext
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_uncounted_exit_is_a_switch(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: define void @loop_contains_store_uncounted_exit_is_a_switch(
+; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: switch i16 [[EE_VAL]], label %[[FOR_INC]] [
+; CHECK-NEXT: i16 500, label %[[EXIT:.*]]
+; CHECK-NEXT: ]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ switch i16 %ee.val, label %for.inc [ i16 500, label %exit ]
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @loop_contains_store_uncounted_exit_is_not_guaranteed_to_execute(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: define void @loop_contains_store_uncounted_exit_is_not_guaranteed_to_execute(
+; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[REM:%.*]] = urem i64 [[IV]], 5
+; CHECK-NEXT: [[SKIP_EE_CMP:%.*]] = icmp eq i64 [[REM]], 0
+; CHECK-NEXT: br i1 [[SKIP_EE_CMP]], label %[[FOR_INC]], label %[[EE_BLOCK:.*]]
+; CHECK: [[EE_BLOCK]]:
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %rem = urem i64 %iv, 5
+ %skip.ee.cmp = icmp eq i64 %rem, 0
+ br i1 %skip.ee.cmp, label %for.inc, label %ee.block
+
+ee.block:
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @test_nodep(ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: define void @test_nodep(
+; CHECK-SAME: ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: store i16 0, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ store i16 0, ptr %st.addr, align 2
+ %ee.val = load i16, ptr %st.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @histogram_with_uncountable_exit(ptr noalias %buckets, ptr readonly %indices, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: define void @histogram_with_uncountable_exit(
+; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], ptr readonly align 2 dereferenceable(40) [[PRED:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[GEP_INDICES:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT: [[L_IDX:%.*]] = load i32, ptr [[GEP_INDICES]], align 4
+; CHECK-NEXT: [[IDXPROM1:%.*]] = zext i32 [[L_IDX]] to i64
+; CHECK-NEXT: [[GEP_BUCKET:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
+; CHECK-NEXT: [[L_BUCKET:%.*]] = load i32, ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[L_BUCKET]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[GEP_BUCKET]], align 4
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[IV]]
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %gep.indices = getelementptr inbounds i32, ptr %indices, i64 %iv
+ %l.idx = load i32, ptr %gep.indices, align 4
+ %idxprom1 = zext i32 %l.idx to i64
+ %gep.bucket = getelementptr inbounds i32, ptr %buckets, i64 %idxprom1
+ %l.bucket = load i32, ptr %gep.bucket, align 4
+ %inc = add nsw i32 %l.bucket, 1
+ store i32 %inc, ptr %gep.bucket, align 4
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @uncountable_exit_condition_address_is_invariant(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(2) readonly %pred) {
+; CHECK-LABEL: define void @uncountable_exit_condition_address_is_invariant(
+; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(2) [[PRED:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[PRED]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT:.*]], label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[EXIT]], label %[[FOR_BODY]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.val = load i16, ptr %pred, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+ ret void
+}
+
+define void @uncountable_exit_condition_address_is_addrec_in_outer_loop(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(2) readonly %pred) {
+; CHECK-LABEL: define void @uncountable_exit_condition_address_is_addrec_in_outer_loop(
+; CHECK-SAME: ptr noalias dereferenceable(40) [[ARRAY:%.*]], ptr readonly align 2 dereferenceable(2) [[PRED:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[OUTER_BODY:.*]]
+; CHECK: [[OUTER_BODY]]:
+; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_INC:.*]] ]
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[PRED]], i64 [[OUTER_IV]]
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[OUTER_BODY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[ST_ADDR:%.*]] = getelementptr inbounds nuw i16, ptr [[ARRAY]], i64 [[IV]]
+; CHECK-NEXT: [[DATA:%.*]] = load i16, ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[INC:%.*]] = add nsw i16 [[DATA]], 1
+; CHECK-NEXT: store i16 [[INC]], ptr [[ST_ADDR]], align 2
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i16, ptr [[EE_ADDR]], align 2
+; CHECK-NEXT: [[EE_COND:%.*]] = icmp sgt i16 [[EE_VAL]], 500
+; CHECK-NEXT: br i1 [[EE_COND]], label %[[EXIT_LOOPEXIT:.*]], label %[[FOR_INC]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 20
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[OUTER_INC]], label %[[FOR_BODY]]
+; CHECK: [[OUTER_INC]]:
+; CHECK-NEXT: [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
+; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 2
+; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT_LOOPEXIT1:.*]], label %[[OUTER_BODY]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[EXIT_LOOPEXIT1]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %outer.body
+
+outer.body:
+ %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.inc ]
+ %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %outer.iv
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %outer.body ], [ %iv.next, %for.inc ]
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
+ store i16 %inc, ptr %st.addr, align 2
+ %ee.val = load i16, ptr %ee.addr, align 2
+ %ee.cond = icmp sgt i16 %ee.val, 500
+ br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 20
+ br i1 %counted.cond, label %outer.inc, label %for.body
+
+outer.inc:
+ %outer.iv.next = add nuw nsw i64 %outer.iv, 1
+ %outer.cond = icmp eq i64 %outer.iv.next, 2
+ br i1 %outer.cond, label %exit, label %outer.body
+
+exit:
+ ret void
+}
+
+;; ICE was caused by assert for the load used in the uncountable exit condition
+;; being guaranteed to execute.
+ at ee.global = external global [4 x i8]
+define void @crash_conditional_load_for_uncountable_exit(ptr dereferenceable(40) noalias %store.area) {
+; CHECK-LABEL: define void @crash_conditional_load_for_uncountable_exit(
+; CHECK-SAME: ptr noalias dereferenceable(40) [[STORE_AREA:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr i8, ptr @ee.global, i64 [[IV]]
+; CHECK-NEXT: br i1 false, label %[[EE_BLOCK:.*]], label %[[INVALID_BLOCK:.*]]
+; CHECK: [[EE_BLOCK]]:
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i8, ptr [[EE_ADDR]], align 1
+; CHECK-NEXT: store i16 0, ptr [[STORE_AREA]], align 2
+; CHECK-NEXT: [[EE_CMP:%.*]] = icmp eq i8 [[EE_VAL]], 0
+; CHECK-NEXT: br i1 [[EE_CMP]], label %[[FOR_INC]], label %[[INVALID_BLOCK]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 10
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[INVALID_BLOCK]], label %[[FOR_BODY]]
+; CHECK: [[INVALID_BLOCK]]:
+; CHECK-NEXT: unreachable
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %ee.addr = getelementptr i8, ptr @ee.global, i64 %iv
+ br i1 false, label %ee.block, label %invalid.block
+
+ee.block:
+ %ee.val = load i8, ptr %ee.addr, align 1
+ store i16 0, ptr %store.area, align 2
+ %ee.cmp = icmp eq i8 %ee.val, 0
+ br i1 %ee.cmp, label %for.inc, label %invalid.block
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 10
+ br i1 %counted.cond, label %invalid.block, label %for.body
+
+invalid.block:
+ unreachable
+}
+
+define void @crash_conditional_load_for_uncountable_exit_argptr(ptr dereferenceable(40) noalias %store.area, ptr dereferenceable(4) %load.area, i1 %skip.cond) {
+; CHECK-LABEL: define void @crash_conditional_load_for_uncountable_exit_argptr(
+; CHECK-SAME: ptr noalias dereferenceable(40) [[STORE_AREA:%.*]], ptr dereferenceable(4) [[LOAD_AREA:%.*]], i1 [[SKIP_COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT: [[EE_ADDR:%.*]] = getelementptr i8, ptr [[LOAD_AREA]], i64 [[IV]]
+; CHECK-NEXT: br i1 [[SKIP_COND]], label %[[EE_BLOCK:.*]], label %[[INVALID_BLOCK:.*]]
+; CHECK: [[EE_BLOCK]]:
+; CHECK-NEXT: [[EE_VAL:%.*]] = load i8, ptr [[EE_ADDR]], align 1
+; CHECK-NEXT: store i16 0, ptr [[STORE_AREA]], align 2
+; CHECK-NEXT: [[EE_CMP:%.*]] = icmp eq i8 [[EE_VAL]], 0
+; CHECK-NEXT: br i1 [[EE_CMP]], label %[[FOR_INC]], label %[[INVALID_BLOCK]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[COUNTED_COND:%.*]] = icmp eq i64 [[IV_NEXT]], 10
+; CHECK-NEXT: br i1 [[COUNTED_COND]], label %[[INVALID_BLOCK]], label %[[FOR_BODY]]
+; CHECK: [[INVALID_BLOCK]]:
+; CHECK-NEXT: unreachable
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %ee.addr = getelementptr i8, ptr %load.area, i64 %iv
+ br i1 %skip.cond, label %ee.block, label %invalid.block
+
+ee.block:
+ %ee.val = load i8, ptr %ee.addr, align 1
+ store i16 0, ptr %store.area, align 2
+ %ee.cmp = icmp eq i8 %ee.val, 0
+ br i1 %ee.cmp, label %for.inc, label %invalid.block
+
+for.inc:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %counted.cond = icmp eq i64 %iv.next, 10
+ br i1 %counted.cond, label %invalid.block, label %for.body
+
+invalid.block:
+ unreachable
+}
+
+
+declare void @init_mem(ptr, i64);
+declare i64 @get_an_unknown_offset();
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
index 55b52299d4331..d8041d926e0a6 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
@@ -1,5 +1,5 @@
; REQUIRES: asserts
-; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -force-vector-width=4 -disable-output 2>&1 | FileCheck %s
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-early-exit-with-side-effect-vectorization -force-vector-width=4 -disable-output 2>&1 | FileCheck %s
define i64 @loop_contains_store(ptr %dest) {
; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store'
@@ -30,7 +30,7 @@ loop.end:
define void @loop_contains_store_condition_load_has_single_user(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_condition_load_has_single_user'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+; CHECK: LV: We can vectorize this loop!
entry:
br label %for.body
@@ -196,7 +196,7 @@ exit:
define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 dereferenceable(40) readonly %load.array, ptr align 2 noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_to_pointer_with_no_deref_info'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+; CHECK: LV: We can vectorize this loop!
entry:
br label %for.body
@@ -275,7 +275,7 @@ exit:
define void @loop_contains_store_to_invariant_location(ptr dereferenceable(40) readonly %array, ptr align 2 dereferenceable(40) readonly %pred, ptr noalias %store_addr) {
; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_to_invariant_location'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+; CHECK: LV: Not vectorizing: Cannot vectorize EE loop with LI stores.
entry:
br label %for.body
@@ -301,21 +301,21 @@ exit:
define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_in_latch_block'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+; CHECK: LV: We can vectorize this loop!
entry:
br label %for.body
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
- %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
- %data = load i16, ptr %st.addr, align 2
- %inc = add nsw i16 %data, 1
%ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
%ee.val = load i16, ptr %ee.addr, align 2
%ee.cond = icmp sgt i16 %ee.val, 500
br i1 %ee.cond, label %exit, label %for.inc
for.inc:
+ %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+ %data = load i16, ptr %st.addr, align 2
+ %inc = add nsw i16 %data, 1
store i16 %inc, ptr %st.addr, align 2
%iv.next = add nuw nsw i64 %iv, 1
%counted.cond = icmp eq i64 %iv.next, 20
@@ -381,7 +381,7 @@ exit:
define void @loop_contains_store_decrementing_iv(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_decrementing_iv'
-; CHECK: LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+; CHECK: LV: We can vectorize this loop!
entry:
br label %for.body
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
index 3a585e958c3f3..53f5414e5d827 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
@@ -75,7 +75,10 @@ class VPlanTestIRBase : public testing::Test {
auto Plan = VPlanTransforms::buildVPlan0(L, *LI, IntegerType::get(*Ctx, 64),
{}, PSE);
- VPlanTransforms::handleEarlyExits(*Plan, HasUncountableExit);
+ // TODO: Add tests for uncountable-with-side-effects.
+ VPlanTransforms::handleEarlyExits(
+ *Plan, HasUncountableExit,
+ /*HasUncountableExitWithSideEffects=*/false);
VPlanTransforms::addMiddleCheck(*Plan, true, false);
VPlanTransforms::createLoopRegions(*Plan);
>From 0520fdc2c696666aa9f6729447bb2b4e0ef8f2d8 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter at arm.com>
Date: Tue, 3 Feb 2026 17:35:24 +0000
Subject: [PATCH 2/2] Comments, fixes, exit handling style enum
---
.../Vectorize/LoopVectorizationLegality.h | 5 ++--
.../Vectorize/LoopVectorizationLegality.cpp | 14 +++++----
.../Transforms/Vectorize/LoopVectorize.cpp | 3 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 30 +++++++++++++++++++
.../Vectorize/VPlanConstruction.cpp | 9 ++++--
.../Transforms/Vectorize/VPlanTransforms.cpp | 25 +++++++++++-----
.../Transforms/Vectorize/VPlanTransforms.h | 5 ++--
.../AArch64/early_exit_with_stores.ll | 3 ++
8 files changed, 73 insertions(+), 21 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 59cc98297c93c..ed0dd7aa62051 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -484,7 +484,7 @@ class LoopVectorizationLegality {
}
const ArrayRef<BasicBlock *> getUncountableExitingBlocks() const {
- return UncountableExitingBlocksVec;
+ return UncountableExitingBlocks;
}
private:
@@ -741,8 +741,7 @@ class LoopVectorizationLegality {
/// Keep track of all the countable and uncountable exiting blocks if
/// the exact backedge taken count is not computable.
SmallVector<BasicBlock *, 4> CountableExitingBlocks;
-
- SmallVector<BasicBlock *, 4> UncountableExitingBlocksVec;
+ SmallVector<BasicBlock *, 4> UncountableExitingBlocks;
/// True if the loop has uncountable early exits.
bool HasUncountableEarlyExit = false;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 978d91ea67338..1e2f3af4305e5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1717,7 +1717,6 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
// Keep a record of all the exiting blocks.
SmallVector<const SCEVPredicate *, 4> Predicates;
- SmallVector<BasicBlock *> UncountableExitingBlocks;
for (BasicBlock *BB : ExitingBlocks) {
const SCEV *EC =
PSE.getSE()->getPredicatedExitCount(TheLoop, BB, &Predicates);
@@ -1874,7 +1873,6 @@ bool LoopVectorizationLegality::isVectorizableEarlyExitLoop() {
<< *SymbolicMaxBTC << '\n');
HasUncountableEarlyExit = true;
UncountableExitWithSideEffects = HasSideEffects;
- UncountableExitingBlocksVec.append(UncountableExitingBlocks);
return true;
}
@@ -2060,9 +2058,15 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
}
// TODO: Remove this restriction once we're sure it's safe to do so.
- // If we're doing full masking we would need to store from the last
- // active lane instead of the last lane according to VF. Doing the last
- // iteration in the scalar loop should be ok, but we need to enforce it.
+ // Handling stores to invariant addresses will be slightly different
+ // based on the approach taken. If we bail out before executing any
+ // lane that would take the uncountable exit, then the store that occurs
+ // in the scalar loop would suffice.
+ //
+ // If we instead handle the lane taking the uncountable exit within the
+ // vectorized loop, then we will have to ensure that we extract the
+ // last active lane at that point in the loop instead of the last lane
+ // of the vector before performing a scalar store.
if (hasUncountableExitWithSideEffects() &&
!LAI->getStoresToInvariantAddresses().empty()) {
LLVM_DEBUG(dbgs() << "LV: Cannot vectorize early exit loops with stores to "
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a3ad2c34ca610..a49eecd49d1a0 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8159,7 +8159,7 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
break;
auto *UncountableBR =
cast<BranchInst>(UncountableExits.front()->getTerminator());
- if (!VPlanTransforms::runPass(
+ if (!RUN_VPLAN_PASS(
VPlanTransforms::handleUncountableExitsWithSideEffects, *Plan,
UncountableBR))
break;
@@ -9832,6 +9832,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
IC = 1;
}
+ // FIXME: Enable interleaving for EE-with-side-effects.
if (InterleaveLoop && LVL.hasUncountableExitWithSideEffects()) {
LLVM_DEBUG(dbgs() << "LV: Not interleaving due to EE with side effects.\n");
IntDiagMsg = {"EEWithSideEffectsPreventsInterleaving",
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index a5f314ac188d8..2f3dcb6ff4970 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -76,6 +76,36 @@ typedef unsigned ID;
using VPlanPtr = std::unique_ptr<VPlan>;
+// Different methods of handling early exits.
+// If one of the masked styles is chosen, all memory operations besides the
+// load(s) required to determine whether an uncountable exit occurred will be
+// masked based on that condition. The difference is then between whether all
+// operations are carried out in the vector loop (which may require the
+// the creation of multiple masks, depending on where side effects occur), or
+// whether the vector loop masks off the lane which would exit and processes the
+// exit in a single iteration of the scalar loop.
+//
+// The UnmaskedHandleLastVectorInScalarLoop style doesn't require masking, but
+// must determine whether there would be an exit in the next vector iteration,
+// and if so lets the scalar loop handle all remaining iterations. This does
+// require replicating the IR for all uncountable exits in a pre-header check
+// block to see if the first vector iteration would encounter the exit, and
+// within the vector loop the values for the next vector iteration must be
+// loaded instead.
+//
+// For ReadOnlyUncountableExitsInVectorLoop, there are no side effects to worry
+// about, so we can instead process any uncountable exit inside the vector
+// body and branch to an appropriate exit block, matching the behaviour of the
+// scalar loop.
+namespace EarlyExitStyleTy {
+enum Option {
+ MaskedHandleLastIterationInScalarLoop = 0,
+ MaskedHandleAllIterationsInVectorLoop,
+ UnmaskedHandleLastVectorInScalarLoop,
+ ReadOnlyUncountableExitsInVectorLoop
+};
+} // namespace EarlyExitStyleTy
+
/// VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
/// A VPBlockBase can be either a VPBasicBlock or a VPRegionBlock.
class LLVM_ABI_FOR_TEST VPBlockBase {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index d7dc4d702dc2b..65dc964fe185d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -893,9 +893,12 @@ void VPlanTransforms::handleEarlyExits(VPlan &Plan,
if (HasUncountableEarlyExit) {
assert(!HandledUncountableEarlyExit &&
"can handle exactly one uncountable early exit");
- handleUncountableEarlyExit(cast<VPBasicBlock>(Pred), EB, Plan,
- cast<VPBasicBlock>(HeaderVPB), LatchVPBB,
- HasUncountableExitWithSideEffects);
+ handleUncountableEarlyExit(
+ cast<VPBasicBlock>(Pred), EB, Plan, cast<VPBasicBlock>(HeaderVPB),
+ LatchVPBB,
+ HasUncountableExitWithSideEffects
+ ? EarlyExitStyleTy::MaskedHandleLastIterationInScalarLoop
+ : EarlyExitStyleTy::ReadOnlyUncountableExitsInVectorLoop);
HandledUncountableEarlyExit = true;
} else {
for (VPRecipeBase &R : EB->phis())
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 4317006534384..ab185b04ff400 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4031,9 +4031,14 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
void VPlanTransforms::handleUncountableEarlyExit(
VPBasicBlock *EarlyExitingVPBB, VPBasicBlock *EarlyExitVPBB, VPlan &Plan,
- VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB, bool HasSideEffects) {
+ VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB,
+ EarlyExitStyleTy::Option Style) {
auto *MiddleVPBB = cast<VPBasicBlock>(LatchVPBB->getSuccessors()[0]);
- if (!HasSideEffects && !EarlyExitVPBB->getSinglePredecessor() &&
+ // Phis do not need to be adjusted if we're dealing with any uncountable exits
+ // in the scalar loop.
+ if ((Style == EarlyExitStyleTy::MaskedHandleAllIterationsInVectorLoop ||
+ Style == EarlyExitStyleTy::ReadOnlyUncountableExitsInVectorLoop) &&
+ !EarlyExitVPBB->getSinglePredecessor() &&
EarlyExitVPBB->getPredecessors()[1] == MiddleVPBB) {
assert(EarlyExitVPBB->getNumPredecessors() == 2 &&
EarlyExitVPBB->getPredecessors()[0] == EarlyExitingVPBB &&
@@ -4069,10 +4074,17 @@ void VPlanTransforms::handleUncountableEarlyExit(
DebugLoc LatchDL = LatchExitingBranch->getDebugLoc();
LatchExitingBranch->eraseFromParent();
- if (HasSideEffects) {
+ // If the early-exit loop has side effects (e.g. stores), then we may want to
+ // handle any uncountable exits in the scalar loop. In order for that to work,
+ // if an uncountable exit is taken we must always enter the scalar loop to
+ // perform the last iteration where the exit occurs. So we can bail out here
+ // after combining the exit conditions into one instead of creating multiple
+ // exit blocks and branches.
+ if (Style == EarlyExitStyleTy::MaskedHandleLastIterationInScalarLoop ||
+ Style == EarlyExitStyleTy::UnmaskedHandleLastVectorInScalarLoop) {
Builder.setInsertPoint(LatchVPBB);
- VPInstruction *Combined = Builder.createNaryOp(
- Instruction::Or, {IsEarlyExitTaken, IsLatchExitTaken}, LatchDL);
+ VPInstruction *Combined =
+ Builder.createOr(IsEarlyExitTaken, IsLatchExitTaken, LatchDL);
Builder.createNaryOp(VPInstruction::BranchOnCond, {Combined});
return;
}
@@ -4253,8 +4265,7 @@ bool VPlanTransforms::handleUncountableExitsWithSideEffects(
// brkb+cntp+whilelo in the loop to just brkb. We need a cntp
// outside the loop instead... maybe copy in CodeGenPrepare?
VPTypeAnalysis TypeInfo(Plan);
- VPValue *ExitIV =
- MiddleBuilder.createNaryOp(Instruction::Add, {IV, FirstActive});
+ VPValue *ExitIV = MiddleBuilder.createAdd(IV, FirstActive);
VPValue *FullTC =
MiddleBuilder.createICmp(CmpInst::ICMP_EQ, ExitIV, Plan.getTripCount());
OldTerminator->setOperand(0, FullTC);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 63da7d299595c..5913c53a5c02b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -308,12 +308,13 @@ struct VPlanTransforms {
/// Update \p Plan to account for the uncountable early exit from \p
/// EarlyExitingVPBB to \p EarlyExitVPBB by introducing a BranchOnTwoConds
/// terminator in the latch that handles the early exit and the latch exit
- /// condition.
+ /// condition, or a BranchOnCond terminator with a combined condition based
+ /// on the style of early exit vectorization.
static void handleUncountableEarlyExit(VPBasicBlock *EarlyExitingVPBB,
VPBasicBlock *EarlyExitVPBB,
VPlan &Plan, VPBasicBlock *HeaderVPBB,
VPBasicBlock *LatchVPBB,
- bool HasSideEffects);
+ EarlyExitStyleTy::Option Style);
/// Update \p Plan to mask memory operations in the loop based on whether
/// the early exit is taken or not.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
index 79569ce923745..0eac7fe6e24a2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_with_stores.ll
@@ -3,6 +3,9 @@
target triple = "aarch64-unknown-linux-gnu"
+;; This currently doesn't vectorize because the load used to determine the
+;; uncountable exit condition has a second user (the store).
+;; TODO: Support extra users where possible.
define i64 @loop_contains_store(ptr %dest) {
; CHECK-LABEL: define i64 @loop_contains_store(
; CHECK-SAME: ptr [[DEST:%.*]]) #[[ATTR0:[0-9]+]] {
More information about the llvm-commits
mailing list