[llvm] ede1a96 - [LV] Vectorize early exit loops with multiple exits. (#174864)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Feb 13 08:44:29 PST 2026
Author: Florian Hahn
Date: 2026-02-13T16:44:23Z
New Revision: ede1a9626b898b1fcddb1aa5f82755a50f16841e
URL: https://github.com/llvm/llvm-project/commit/ede1a9626b898b1fcddb1aa5f82755a50f16841e
DIFF: https://github.com/llvm/llvm-project/commit/ede1a9626b898b1fcddb1aa5f82755a50f16841e.diff
LOG: [LV] Vectorize early exit loops with multiple exits. (#174864)
Building on top of the recent changes to introduce BranchOnTwoConds,
this patch adds support for vectorizing loops with multiple early exits,
all dominating a countable latch. The early exits must form a
dominance chain, so we can simply check which early exit has been taken
in dominance order.
Currently LoopVectorizationLegality ensures that all exits other than
the latch must be uncountable. handleUncountableEarlyExits now collects
those uncountable exits and processes each exit.
In the vector region, we compute if any exit has been taken, by taking
the OR of all early exit conditions (EarlyExitConds) and checking if
there's
any active lane.
If the early exit is taken, we exit the loop and compute which early
exit
has been taken. The first taken early exit is the one where its exit
condition is true in the first active lane of EarlyExitConds.
We create a chain of dispatch blocks outside the loop to check this for
the early exit blocks ordered by dominance.
Depends on https://github.com/llvm/llvm-project/pull/174016.
PR: https://github.com/llvm/llvm-project/pull/174864
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.h
llvm/test/Transforms/LoopVectorize/VPlan/uncountable-early-exit-vplan.ll
llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
llvm/test/Transforms/LoopVectorize/multi_early_exit.ll
llvm/test/Transforms/LoopVectorize/multiple-early-exits.ll
llvm/test/Transforms/LoopVectorize/unsupported_early_exit.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 0b8796f646ae3..54bb073eb4f81 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -233,6 +233,12 @@ class VPBuilder {
return createNaryOp(VPInstruction::LogicalAnd, {LHS, RHS}, DL, Name);
}
+ VPInstruction *createLogicalOr(VPValue *LHS, VPValue *RHS,
+ DebugLoc DL = DebugLoc::getUnknown(),
+ const Twine &Name = "") {
+ return createNaryOp(VPInstruction::LogicalOr, {LHS, RHS}, DL, Name);
+ }
+
VPInstruction *createSelect(VPValue *Cond, VPValue *TrueVal,
VPValue *FalseVal,
DebugLoc DL = DebugLoc::getUnknown(),
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 5c3049b54c2de..499c5a31421ed 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9509,15 +9509,6 @@ bool LoopVectorizePass::processLoop(Loop *L) {
"UncountableEarlyExitLoopsDisabled", ORE, L);
return false;
}
- SmallVector<BasicBlock *, 8> ExitingBlocks;
- L->getExitingBlocks(ExitingBlocks);
- // TODO: Support multiple uncountable early exits.
- if (ExitingBlocks.size() - LVL.getCountableExitingBlocks().size() > 1) {
- reportVectorizationFailure("Auto-vectorization of loops with multiple "
- "uncountable early exits is not yet supported",
- "MultipleUncountableEarlyExits", ORE, L);
- return false;
- }
}
if (!LVL.getPotentiallyFaultingLoads().empty()) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 68ff4e6a037f5..07b6c306fa980 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1211,6 +1211,7 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// during unrolling.
ExtractPenultimateElement,
LogicalAnd, // Non-poison propagating logical And.
+ LogicalOr, // Non-poison propagating logical Or.
// Add an offset in bytes (second operand) to a base pointer (first
// operand). Only generates scalar values (either for the first lane only or
// for all lanes, depending on its uses).
@@ -1520,6 +1521,9 @@ class VPPhiAccessors {
/// Returns the incoming block with index \p Idx.
const VPBasicBlock *getIncomingBlock(unsigned Idx) const;
+ /// Returns the incoming value for \p VPBB. \p VPBB must be an incoming block.
+ VPValue *getIncomingValueForBlock(const VPBasicBlock *VPBB) const;
+
/// Returns the number of incoming values, also number of incoming blocks.
virtual unsigned getNumIncoming() const {
return getAsRecipe()->getNumOperands();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 5bd4f6b8026b8..35f4f7c1a48ec 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -124,9 +124,10 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
case VPInstruction::LastActiveLane:
return Type::getIntNTy(Ctx, 64);
case VPInstruction::LogicalAnd:
+ case VPInstruction::LogicalOr:
assert(inferScalarType(R->getOperand(0))->isIntegerTy(1) &&
inferScalarType(R->getOperand(1))->isIntegerTy(1) &&
- "LogicalAnd operands should be bool");
+ "LogicalAnd/Or operands should be bool");
return IntegerType::get(Ctx, 1);
case VPInstruction::BranchOnCond:
case VPInstruction::BranchOnTwoConds:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 9b4ae56e7f175..da1035847020a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -903,33 +903,27 @@ void VPlanTransforms::handleEarlyExits(VPlan &Plan,
auto *LatchVPBB = cast<VPBasicBlock>(MiddleVPBB->getSinglePredecessor());
VPBlockBase *HeaderVPB = cast<VPBasicBlock>(LatchVPBB->getSuccessors()[1]);
- // Disconnect all early exits from the loop leaving it with a single exit from
- // the latch. Early exits that are countable are left for a scalar epilog. The
- // condition of uncountable early exits (currently at most one is supported)
- // is fused into the latch exit, and used to branch from middle block to the
- // early exit destination.
- [[maybe_unused]] bool HandledUncountableEarlyExit = false;
+ if (HasUncountableEarlyExit) {
+ handleUncountableEarlyExits(Plan, cast<VPBasicBlock>(HeaderVPB), LatchVPBB,
+ MiddleVPBB);
+ return;
+ }
+
+ // Disconnect countable early exits from the loop, leaving it with a single
+ // exit from the latch. Countable early exits are left for a scalar epilog.
for (VPIRBasicBlock *EB : Plan.getExitBlocks()) {
for (VPBlockBase *Pred : to_vector(EB->getPredecessors())) {
if (Pred == MiddleVPBB)
continue;
- if (HasUncountableEarlyExit) {
- assert(!HandledUncountableEarlyExit &&
- "can handle exactly one uncountable early exit");
- handleUncountableEarlyExit(cast<VPBasicBlock>(Pred), EB, Plan,
- cast<VPBasicBlock>(HeaderVPB), LatchVPBB);
- HandledUncountableEarlyExit = true;
- } else {
- for (VPRecipeBase &R : EB->phis())
- cast<VPIRPhi>(&R)->removeIncomingValueFor(Pred);
- }
- cast<VPBasicBlock>(Pred)->getTerminator()->eraseFromParent();
+
+ // Remove phi operands for the early exiting block.
+ for (VPRecipeBase &R : EB->phis())
+ cast<VPIRPhi>(&R)->removeIncomingValueFor(Pred);
+ auto *EarlyExitingVPBB = cast<VPBasicBlock>(Pred);
+ EarlyExitingVPBB->getTerminator()->eraseFromParent();
VPBlockUtils::disconnectBlocks(Pred, EB);
}
}
-
- assert((!HasUncountableEarlyExit || HandledUncountableEarlyExit) &&
- "missed an uncountable exit that must be handled");
}
void VPlanTransforms::addMiddleCheck(VPlan &Plan,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 34cb5a8b48f78..5154e0e607eb4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -833,9 +833,10 @@ inline auto m_c_LogicalAnd(const Op0_t &Op0, const Op1_t &Op1) {
}
template <typename Op0_t, typename Op1_t>
-inline AllRecipe_match<Instruction::Select, Op0_t, specific_intval<1>, Op1_t>
-m_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) {
- return m_Select(Op0, m_True(), Op1);
+inline auto m_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) {
+ return m_CombineOr(
+ m_c_VPInstruction<VPInstruction::LogicalOr, Op0_t, Op1_t>(Op0, Op1),
+ m_Select(Op0, m_True(), Op1));
}
template <typename Op0_t, typename Op1_t>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 89a270d1219e7..389331fd958d2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -468,6 +468,7 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case VPInstruction::ExitingIVValue:
case VPInstruction::FirstOrderRecurrenceSplice:
case VPInstruction::LogicalAnd:
+ case VPInstruction::LogicalOr:
case VPInstruction::PtrAdd:
case VPInstruction::WidePtrAdd:
case VPInstruction::WideIVStep:
@@ -813,6 +814,11 @@ Value *VPInstruction::generate(VPTransformState &State) {
Value *B = State.get(getOperand(1));
return Builder.CreateLogicalAnd(A, B, Name);
}
+ case VPInstruction::LogicalOr: {
+ Value *A = State.get(getOperand(0));
+ Value *B = State.get(getOperand(1));
+ return Builder.CreateLogicalOr(A, B, Name);
+ }
case VPInstruction::PtrAdd: {
assert((State.VF.isScalar() || vputils::onlyFirstLaneUsed(this)) &&
"can only generate first lane for PtrAdd");
@@ -1339,6 +1345,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
case VPInstruction::ExtractLastActive:
case VPInstruction::FirstOrderRecurrenceSplice:
case VPInstruction::LogicalAnd:
+ case VPInstruction::LogicalOr:
case VPInstruction::Not:
case VPInstruction::PtrAdd:
case VPInstruction::WideIVStep:
@@ -1506,6 +1513,9 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::LogicalAnd:
O << "logical-and";
break;
+ case VPInstruction::LogicalOr:
+ O << "logical-or";
+ break;
case VPInstruction::PtrAdd:
O << "ptradd";
break;
@@ -1705,6 +1715,14 @@ void VPPhiAccessors::removeIncomingValueFor(VPBlockBase *IncomingBlock) const {
R->removeOperand(Position);
}
+VPValue *
+VPPhiAccessors::getIncomingValueForBlock(const VPBasicBlock *VPBB) const {
+ for (unsigned Idx = 0; Idx != getNumIncoming(); ++Idx)
+ if (getIncomingBlock(Idx) == VPBB)
+ return getIncomingValue(Idx);
+ llvm_unreachable("VPBB is not an incoming block");
+}
+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPPhiAccessors::printPhiOperands(raw_ostream &O,
VPSlotTracker &SlotTracker) const {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b05d8adff8a2e..da3afe7ce6d03 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4036,75 +4036,211 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
R->eraseFromParent();
}
-void VPlanTransforms::handleUncountableEarlyExit(VPBasicBlock *EarlyExitingVPBB,
- VPBasicBlock *EarlyExitVPBB,
- VPlan &Plan,
- VPBasicBlock *HeaderVPBB,
- VPBasicBlock *LatchVPBB) {
- auto *MiddleVPBB = cast<VPBasicBlock>(LatchVPBB->getSuccessors()[0]);
- if (!EarlyExitVPBB->getSinglePredecessor() &&
- EarlyExitVPBB->getPredecessors()[1] == MiddleVPBB) {
- assert(EarlyExitVPBB->getNumPredecessors() == 2 &&
- EarlyExitVPBB->getPredecessors()[0] == EarlyExitingVPBB &&
- "unsupported early exit VPBB");
- // Early exit operand should always be last phi operand. If EarlyExitVPBB
- // has two predecessors and EarlyExitingVPBB is the first, swap the operands
- // of the phis.
- for (VPRecipeBase &R : EarlyExitVPBB->phis())
- cast<VPIRPhi>(&R)->swapOperands();
- }
+void VPlanTransforms::handleUncountableEarlyExits(VPlan &Plan,
+ VPBasicBlock *HeaderVPBB,
+ VPBasicBlock *LatchVPBB,
+ VPBasicBlock *MiddleVPBB) {
+ struct EarlyExitInfo {
+ VPBasicBlock *EarlyExitingVPBB;
+ VPIRBasicBlock *EarlyExitVPBB;
+ VPValue *CondToExit;
+ };
+ VPDominatorTree VPDT(Plan);
VPBuilder Builder(LatchVPBB->getTerminator());
- VPBlockBase *TrueSucc = EarlyExitingVPBB->getSuccessors()[0];
- assert(match(EarlyExitingVPBB->getTerminator(), m_BranchOnCond()) &&
- "Terminator must be be BranchOnCond");
- VPValue *CondOfEarlyExitingVPBB =
- EarlyExitingVPBB->getTerminator()->getOperand(0);
- auto *CondToEarlyExit = TrueSucc == EarlyExitVPBB
- ? CondOfEarlyExitingVPBB
- : Builder.createNot(CondOfEarlyExitingVPBB);
-
- // Create a BranchOnTwoConds in the latch that branches to:
- // [0] vector.early.exit, [1] middle block, [2] header (continue looping).
- VPValue *IsEarlyExitTaken =
- Builder.createNaryOp(VPInstruction::AnyOf, {CondToEarlyExit});
- VPBasicBlock *VectorEarlyExitVPBB =
- Plan.createVPBasicBlock("vector.early.exit");
- VectorEarlyExitVPBB->setParent(EarlyExitVPBB->getParent());
-
- VPBlockUtils::connectBlocks(VectorEarlyExitVPBB, EarlyExitVPBB);
-
- // Update the exit phis in the early exit block.
- VPBuilder MiddleBuilder(MiddleVPBB);
- VPBuilder EarlyExitB(VectorEarlyExitVPBB);
- for (VPRecipeBase &R : EarlyExitVPBB->phis()) {
- auto *ExitIRI = cast<VPIRPhi>(&R);
- // Early exit operand should always be last, i.e., 0 if EarlyExitVPBB has
- // a single predecessor and 1 if it has two.
- unsigned EarlyExitIdx = ExitIRI->getNumOperands() - 1;
- if (ExitIRI->getNumOperands() != 1) {
- // The first of two operands corresponds to the latch exit, via MiddleVPBB
- // predecessor. Extract its final lane.
- ExitIRI->extractLastLaneOfLastPartOfFirstOperand(MiddleBuilder);
+ SmallVector<EarlyExitInfo> Exits;
+ for (VPIRBasicBlock *ExitBlock : Plan.getExitBlocks()) {
+ for (VPBlockBase *Pred : to_vector(ExitBlock->getPredecessors())) {
+ if (Pred == MiddleVPBB)
+ continue;
+ // Collect condition for this early exit.
+ auto *EarlyExitingVPBB = cast<VPBasicBlock>(Pred);
+ VPBlockBase *TrueSucc = EarlyExitingVPBB->getSuccessors()[0];
+ VPValue *CondOfEarlyExitingVPBB;
+ [[maybe_unused]] bool Matched =
+ match(EarlyExitingVPBB->getTerminator(),
+ m_BranchOnCond(m_VPValue(CondOfEarlyExitingVPBB)));
+ assert(Matched && "Terminator must be BranchOnCond");
+ auto *CondToEarlyExit = TrueSucc == ExitBlock
+ ? CondOfEarlyExitingVPBB
+ : Builder.createNot(CondOfEarlyExitingVPBB);
+ assert((isa<VPIRValue>(CondOfEarlyExitingVPBB) ||
+ VPDT.properlyDominates(
+ CondOfEarlyExitingVPBB->getDefiningRecipe()->getParent(),
+ LatchVPBB)) &&
+ "exit condition must dominate the latch");
+ Exits.push_back({
+ EarlyExitingVPBB,
+ ExitBlock,
+ CondToEarlyExit,
+ });
+ }
+ }
+
+ assert(!Exits.empty() && "must have at least one early exit");
+ // Sort exits by dominance to get the correct program order.
+ llvm::sort(Exits, [&VPDT](const EarlyExitInfo &A, const EarlyExitInfo &B) {
+ return VPDT.dominates(A.EarlyExitingVPBB, B.EarlyExitingVPBB);
+ });
+
+ // Build the AnyOf condition for the latch terminator using logical OR
+ // to avoid poison propagation from later exit conditions when an earlier
+ // exit is taken.
+ VPValue *Combined = Exits[0].CondToExit;
+ for (const EarlyExitInfo &Info : drop_begin(Exits))
+ Combined = Builder.createLogicalOr(Combined, Info.CondToExit);
+
+ VPValue *IsAnyExitTaken =
+ Builder.createNaryOp(VPInstruction::AnyOf, {Combined});
+
+ // Create the vector.early.exit blocks.
+ SmallVector<VPBasicBlock *> VectorEarlyExitVPBBs(Exits.size());
+ for (unsigned Idx = 0; Idx != Exits.size(); ++Idx) {
+ Twine BlockSuffix = Exits.size() == 1 ? "" : Twine(".") + Twine(Idx);
+ VPBasicBlock *VectorEarlyExitVPBB =
+ Plan.createVPBasicBlock("vector.early.exit" + BlockSuffix);
+ VectorEarlyExitVPBBs[Idx] = VectorEarlyExitVPBB;
+ }
+
+ // Create the dispatch block (or reuse the single exit block if only one
+ // exit). The dispatch block computes the first active lane of the combined
+ // condition and, for multiple exits, chains through conditions to determine
+ // which exit to take.
+ VPBasicBlock *DispatchVPBB =
+ Exits.size() == 1 ? VectorEarlyExitVPBBs[0]
+ : Plan.createVPBasicBlock("vector.early.exit.check");
+ VPBuilder DispatchBuilder(DispatchVPBB, DispatchVPBB->begin());
+ VPValue *FirstActiveLane =
+ DispatchBuilder.createNaryOp(VPInstruction::FirstActiveLane, {Combined},
+ DebugLoc::getUnknown(), "first.active.lane");
+
+ // For each early exit, disconnect the original exiting block
+ // (early.exiting.I) from the exit block (ir-bb<exit.I>) and route through a
+ // new vector.early.exit block. Update ir-bb<exit.I>'s phis to extract their
+ // values at the first active lane:
+ //
+ // Input:
+ // early.exiting.I:
+ // ...
+ // EMIT branch-on-cond vp<%cond.I>
+ // Successor(s): in.loop.succ, ir-bb<exit.I>
+ //
+ // ir-bb<exit.I>:
+ // IR %phi = phi [ vp<%incoming.I>, early.exiting.I ], ...
+ //
+ // Output:
+ // early.exiting.I:
+ // ...
+ // Successor(s): in.loop.succ
+ //
+ // vector.early.exit.I:
+ // EMIT vp<%exit.val> = extract-lane vp<%first.lane>, vp<%incoming.I>
+ // Successor(s): ir-bb<exit.I>
+ //
+ // ir-bb<exit.I>:
+ // IR %phi = phi ... (extra operand: vp<%exit.val> from
+ // vector.early.exit.I)
+ //
+ for (auto [Exit, VectorEarlyExitVPBB] :
+ zip_equal(Exits, VectorEarlyExitVPBBs)) {
+ auto &[EarlyExitingVPBB, EarlyExitVPBB, _] = Exit;
+ // Adjust the phi nodes in EarlyExitVPBB.
+ // 1. remove incoming values from EarlyExitingVPBB,
+ // 2. extract the incoming value at FirstActiveLane
+ // 3. add back the extracts as last operands for the phis
+ // Then adjust the CFG, removing the edge between EarlyExitingVPBB and
+ // EarlyExitVPBB and adding a new edge between VectorEarlyExitVPBB and
+ // EarlyExitVPBB. The extracts at FirstActiveLane are now the incoming
+ // values from VectorEarlyExitVPBB.
+ for (VPRecipeBase &R : EarlyExitVPBB->phis()) {
+ auto *ExitIRI = cast<VPIRPhi>(&R);
+ VPValue *IncomingVal =
+ ExitIRI->getIncomingValueForBlock(EarlyExitingVPBB);
+ VPValue *NewIncoming = IncomingVal;
+ if (!isa<VPIRValue>(IncomingVal)) {
+ VPBuilder EarlyExitBuilder(VectorEarlyExitVPBB);
+ NewIncoming = EarlyExitBuilder.createNaryOp(
+ VPInstruction::ExtractLane, {FirstActiveLane, IncomingVal},
+ DebugLoc::getUnknown(), "early.exit.value");
+ }
+ ExitIRI->removeIncomingValueFor(EarlyExitingVPBB);
+ ExitIRI->addOperand(NewIncoming);
}
- VPValue *IncomingFromEarlyExit = ExitIRI->getOperand(EarlyExitIdx);
- if (!isa<VPIRValue>(IncomingFromEarlyExit)) {
- // Update the incoming value from the early exit.
- VPValue *FirstActiveLane = EarlyExitB.createNaryOp(
- VPInstruction::FirstActiveLane, {CondToEarlyExit},
- DebugLoc::getUnknown(), "first.active.lane");
- IncomingFromEarlyExit = EarlyExitB.createNaryOp(
- VPInstruction::ExtractLane, {FirstActiveLane, IncomingFromEarlyExit},
- DebugLoc::getUnknown(), "early.exit.value");
- ExitIRI->setOperand(EarlyExitIdx, IncomingFromEarlyExit);
+ EarlyExitingVPBB->getTerminator()->eraseFromParent();
+ VPBlockUtils::disconnectBlocks(EarlyExitingVPBB, EarlyExitVPBB);
+ VPBlockUtils::connectBlocks(VectorEarlyExitVPBB, EarlyExitVPBB);
+ }
+
+ // For exit blocks that also have the middle block as predecessor (latch
+ // exits to the same block as an early exit), extract the last lane of the
+ // first operand for the middle block's incoming value.
+ VPBuilder MiddleBuilder(MiddleVPBB);
+ VPBasicBlock *MiddleSuccVPBB =
+ cast<VPIRBasicBlock>(MiddleVPBB->getSuccessors()[0]);
+ if (MiddleSuccVPBB->getNumPredecessors() > 1) {
+ assert(all_of(MiddleSuccVPBB->getPredecessors(),
+ [&](VPBlockBase *Pred) {
+ return Pred == MiddleVPBB ||
+ is_contained(VectorEarlyExitVPBBs, Pred);
+ }) &&
+ "All predecessors must be either the middle block or early exit "
+ "blocks");
+
+ for (VPRecipeBase &R : MiddleSuccVPBB->phis()) {
+ auto *ExitIRI = cast<VPIRPhi>(&R);
+ assert(ExitIRI->getIncomingValueForBlock(MiddleVPBB) ==
+ ExitIRI->getOperand(0) &&
+ "First operand must come from middle block");
+ ExitIRI->extractLastLaneOfLastPartOfFirstOperand(MiddleBuilder);
}
}
- // Replace the conditional branch controlling the latch exit from the vector
- // loop with a multi-conditional branch exiting to vector early exit if the
- // early exit has been taken, exiting to middle block if the original
- // condition of the vector latch is true, otherwise continuing back to header.
+ // Chain through exits: for each exit, check if its condition is true at
+ // the first active lane. If so, take that exit; otherwise, try the next.
+ // The last exit needs no check since it must be taken if all others fail.
+ //
+ // For 3 exits (cond.0, cond.1, cond.2), this creates:
+ //
+ // latch:
+ // ...
+ // EMIT vp<%combined> = logical-or vp<%cond.0>, vp<%cond.1>, vp<%cond.2>
+ // ...
+ //
+ // vector.early.exit.check:
+ // EMIT vp<%first.lane> = first-active-lane vp<%combined>
+ // EMIT vp<%at.cond.0> = extract-lane vp<%first.lane>, vp<%cond.0>
+ // EMIT branch-on-cond vp<%at.cond.0>
+ // Successor(s): vector.early.exit.0, vector.early.exit.check.0
+ //
+ // vector.early.exit.check.0:
+ // EMIT vp<%at.cond.1> = extract-lane vp<%first.lane>, vp<%cond.1>
+ // EMIT branch-on-cond vp<%at.cond.1>
+ // Successor(s): vector.early.exit.1, vector.early.exit.2
+ VPBasicBlock *CurrentBB = DispatchVPBB;
+ for (auto [I, Exit] : enumerate(ArrayRef(Exits).drop_back())) {
+ VPValue *LaneVal = DispatchBuilder.createNaryOp(
+ VPInstruction::ExtractLane, {FirstActiveLane, Exit.CondToExit},
+ DebugLoc::getUnknown(), "exit.cond.at.lane");
+
+ // For the last dispatch, branch directly to the last exit on false;
+ // otherwise, create a new check block.
+ bool IsLastDispatch = (I + 2 == Exits.size());
+ VPBasicBlock *FalseBB =
+ IsLastDispatch ? VectorEarlyExitVPBBs.back()
+ : Plan.createVPBasicBlock(
+ Twine("vector.early.exit.check.") + Twine(I));
+
+ DispatchBuilder.createNaryOp(VPInstruction::BranchOnCond, {LaneVal});
+ CurrentBB->setSuccessors({VectorEarlyExitVPBBs[I], FalseBB});
+ VectorEarlyExitVPBBs[I]->setPredecessors({CurrentBB});
+ FalseBB->setPredecessors({CurrentBB});
+
+ CurrentBB = FalseBB;
+ DispatchBuilder.setInsertPoint(CurrentBB);
+ }
+
+ // Replace the latch terminator with the new branching logic.
auto *LatchExitingBranch = cast<VPInstruction>(LatchVPBB->getTerminator());
assert(LatchExitingBranch->getOpcode() == VPInstruction::BranchOnCount &&
"Unexpected terminator");
@@ -4114,13 +4250,12 @@ void VPlanTransforms::handleUncountableEarlyExit(VPBasicBlock *EarlyExitingVPBB,
DebugLoc LatchDL = LatchExitingBranch->getDebugLoc();
LatchExitingBranch->eraseFromParent();
-
Builder.setInsertPoint(LatchVPBB);
Builder.createNaryOp(VPInstruction::BranchOnTwoConds,
- {IsEarlyExitTaken, IsLatchExitTaken}, LatchDL);
+ {IsAnyExitTaken, IsLatchExitTaken}, LatchDL);
LatchVPBB->clearSuccessors();
- LatchVPBB->setSuccessors({VectorEarlyExitVPBB, MiddleVPBB, HeaderVPBB});
- VectorEarlyExitVPBB->setPredecessors({LatchVPBB});
+ LatchVPBB->setSuccessors({DispatchVPBB, MiddleVPBB, HeaderVPBB});
+ DispatchVPBB->setPredecessors({LatchVPBB});
}
/// This function tries convert extended in-loop reductions to
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index edf21115e2bab..5fc68c2df145f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -311,14 +311,13 @@ struct VPlanTransforms {
/// Remove dead recipes from \p Plan.
static void removeDeadRecipes(VPlan &Plan);
- /// Update \p Plan to account for the uncountable early exit from \p
- /// EarlyExitingVPBB to \p EarlyExitVPBB by introducing a BranchOnTwoConds
- /// terminator in the latch that handles the early exit and the latch exit
- /// condition.
- static void handleUncountableEarlyExit(VPBasicBlock *EarlyExitingVPBB,
- VPBasicBlock *EarlyExitVPBB,
- VPlan &Plan, VPBasicBlock *HeaderVPBB,
- VPBasicBlock *LatchVPBB);
+ /// Update \p Plan to account for uncountable early exits by introducing
+ /// appropriate branching logic in the latch that handles early exits and the
+ /// latch exit condition. Multiple exits are handled with a dispatch block
+ /// that determines which exit to take based on lane-by-lane semantics.
+ static void handleUncountableEarlyExits(VPlan &Plan, VPBasicBlock *HeaderVPBB,
+ VPBasicBlock *LatchVPBB,
+ VPBasicBlock *MiddleVPBB);
/// Replaces the exit condition from
/// (branch-on-cond eq CanonicalIVInc, VectorTripCount)
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/uncountable-early-exit-vplan.ll b/llvm/test/Transforms/LoopVectorize/VPlan/uncountable-early-exit-vplan.ll
index fbe8756e34ab8..a163f091a0ba4 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/uncountable-early-exit-vplan.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/uncountable-early-exit-vplan.ll
@@ -251,7 +251,64 @@ exit:
}
define i64 @two_early_exits_same_exit_with_constant_live_outs() {
-; CHECK: LV: Not vectorizing: Auto-vectorization of loops with multiple uncountable early exits is not yet supported.
+; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<67> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: IR %A = alloca [1024 x i8], align 1
+; CHECK-NEXT: IR %B = alloca [1024 x i8], align 1
+; CHECK-NEXT: IR call void @init(ptr %A, i64 1024)
+; CHECK-NEXT: IR call void @init(ptr %B, i64 1024)
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK-NEXT: vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>, vp<[[VF]]>
+; CHECK-NEXT: CLONE ir<%gep.A> = getelementptr inbounds ir<%A>, vp<[[SCALAR_STEPS]]>
+; CHECK-NEXT: vp<[[PTRA:%.+]]> = vector-pointer inbounds ir<%gep.A>
+; CHECK-NEXT: WIDEN ir<%ld.A> = load vp<[[PTRA]]>
+; CHECK-NEXT: WIDEN ir<%cmp1> = icmp eq ir<%ld.A>, ir<42>
+; CHECK-NEXT: CLONE ir<%gep.B> = getelementptr inbounds ir<%B>, vp<[[SCALAR_STEPS]]>
+; CHECK-NEXT: vp<[[PTRB:%.+]]> = vector-pointer inbounds ir<%gep.B>
+; CHECK-NEXT: WIDEN ir<%ld.B> = load vp<[[PTRB]]>
+; CHECK-NEXT: WIDEN ir<%cmp2> = icmp eq ir<%ld.A>, ir<%ld.B>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
+; CHECK-NEXT: EMIT vp<[[OR:%.+]]> = logical-or ir<%cmp1>, ir<%cmp2>
+; CHECK-NEXT: EMIT vp<[[ANY_OF:%.+]]> = any-of vp<[[OR]]>
+; CHECK-NEXT: EMIT vp<[[CMP:%.+]]> = icmp eq vp<%index.next>, vp<[[VTC]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[ANY_OF]]>, vp<[[CMP]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): vector.early.exit.check, middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<67>, vp<[[VTC]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.early.exit.check:
+; CHECK-NEXT: EMIT vp<%first.active.lane> = first-active-lane vp<[[OR]]>
+; CHECK-NEXT: EMIT vp<%exit.cond.at.lane> = extract-lane vp<%first.active.lane>, ir<%cmp1>
+; CHECK-NEXT: EMIT branch-on-cond vp<%exit.cond.at.lane>
+; CHECK-NEXT: Successor(s): vector.early.exit.0, vector.early.exit.1
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.early.exit.1:
+; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.early.exit.0:
+; CHECK-NEXT: EMIT vp<[[FIRST_ACTIVE:%.+]]> = first-active-lane vp<[[OR]]>
+; CHECK-NEXT: EMIT vp<[[FINAL_IV:%.+]]> = add vp<[[CAN_IV]]>, vp<[[FIRST_ACTIVE]]>
+; CHECK-NEXT: Successor(s): ir-bb<exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %retval = phi i64 [ %iv, %loop.header ], [ 100, %early.exit.0 ], [ 43, %loop.latch ] (extra operands: ir<43> from middle.block, vp<[[FINAL_IV]]> from vector.early.exit.0, ir<100> from vector.early.exit.1)
;
entry:
%A = alloca [1024 x i8]
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
index c6abc932b653a..2b68a4787b15a 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
@@ -346,12 +346,10 @@ loop.end:
}
-; Multiple uncountable early exits pass legality but are not yet supported
-; in VPlan transformations.
+; Multiple uncountable early exits are now supported.
define i64 @multiple_uncountable_exits() {
; CHECK-LABEL: LV: Checking a loop in 'multiple_uncountable_exits'
; CHECK: LV: We can vectorize this loop!
-; CHECK: LV: Not vectorizing: Auto-vectorization of loops with multiple uncountable early exits is not yet supported.
entry:
%p1 = alloca [1024 x i8]
%p2 = alloca [1024 x i8]
diff --git a/llvm/test/Transforms/LoopVectorize/multi_early_exit.ll b/llvm/test/Transforms/LoopVectorize/multi_early_exit.ll
index 94af5b7c7607d..239fc03d79798 100644
--- a/llvm/test/Transforms/LoopVectorize/multi_early_exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/multi_early_exit.ll
@@ -120,3 +120,73 @@ loop.end.early:
loop.end:
ret i64 0
}
+
+; Test that early exit with a live-in condition (function argument) combined
+; with a dereferenceable load-based condition does not crash. The live-in
+; condition has no defining recipe in VPlan.
+define i64 @early_exit_with_live_in_condition(i1 %cond) {
+; CHECK-LABEL: define i64 @early_exit_with_live_in_condition(
+; CHECK-SAME: i1 [[COND:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[P:%.*]] = alloca [1024 x i8], align 1
+; CHECK-NEXT: call void @init_mem(ptr [[P]], i64 1024)
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[COND]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, [[LOOP]] ], [ [[INC:%.*]], [[VECTOR_BODY_INTERIM:%.*]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[INC]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i1> splat (i1 true), <4 x i1> [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]])
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INC]], 1024
+; CHECK-NEXT: br i1 [[TMP4]], label [[VECTOR_EARLY_EXIT_CHECK:%.*]], label [[VECTOR_BODY_INTERIM]]
+; CHECK: vector.body.interim:
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: middle.block:
+; CHECK-NEXT: br label [[LOOP_END:%.*]]
+; CHECK: vector.early.exit.check:
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[BROADCAST_SPLAT]], i32 0
+; CHECK-NEXT: br i1 [[TMP5]], label [[VECTOR_EARLY_EXIT_0:%.*]], label [[VECTOR_EARLY_EXIT_1:%.*]]
+; CHECK: vector.early.exit.1:
+; CHECK-NEXT: br label [[LOOP_END]]
+; CHECK: vector.early.exit.0:
+; CHECK-NEXT: br label [[LOOP_END]]
+; CHECK: loop.end:
+; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ 1, [[VECTOR_EARLY_EXIT_1]] ], [ 0, [[VECTOR_EARLY_EXIT_0]] ], [ 2, [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: ret i64 [[RETVAL]]
+;
+entry:
+ %p = alloca [1024 x i8]
+ call void @init_mem(ptr %p, i64 1024)
+ br label %loop
+
+loop:
+ %iv = phi i64 [ %inc, %latch ], [ 0, %entry ]
+ br i1 %cond, label %loop.end, label %search
+
+search:
+ %arrayidx = getelementptr inbounds i8, ptr %p, i64 %iv
+ %ld = load i8, ptr %arrayidx, align 1
+ %cmp = icmp eq i8 %ld, 0
+ br i1 %cmp, label %loop.end, label %latch
+
+latch:
+ %inc = add nuw nsw i64 %iv, 1
+ %exitcond = icmp eq i64 %inc, 1024
+ br i1 %exitcond, label %loop.end, label %loop
+
+loop.end:
+ %retval = phi i64 [ 0, %loop ], [ 1, %search ], [ 2, %latch ]
+ ret i64 %retval
+}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/multiple-early-exits.ll b/llvm/test/Transforms/LoopVectorize/multiple-early-exits.ll
index 75cdfe2b9cdd5..4cabc711f5b7d 100644
--- a/llvm/test/Transforms/LoopVectorize/multiple-early-exits.ll
+++ b/llvm/test/Transforms/LoopVectorize/multiple-early-exits.ll
@@ -7,29 +7,60 @@ declare void @init_mem(ptr, i64)
define i64 @two_early_exits_same_exit() {
; CHECK-LABEL: define i64 @two_early_exits_same_exit() {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT1:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[GEP_A]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], splat (i8 42)
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[INDEX_NEXT1]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT1]], 64
+; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 false)
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP1]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
-; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 64, %[[SCALAR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV1]]
+; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[GEP_A1]], align 1
; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LD1]], 42
-; CHECK-NEXT: br i1 [[CMP1]], label %[[EXIT:.*]], label %[[EARLY_EXIT_0:.*]]
+; CHECK-NEXT: br i1 [[CMP1]], label %[[EXIT]], label %[[EARLY_EXIT_0:.*]]
; CHECK: [[EARLY_EXIT_0]]:
-; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV1]]
; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
; CHECK-NEXT: [[CMP2:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
; CHECK-NEXT: br i1 [[CMP2]], label %[[EXIT]], label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP_HEADER]], label %[[EXIT]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP_HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ], [ 100, %[[EARLY_EXIT_0]] ], [ 43, %[[LOOP_LATCH]] ]
+; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[IV1]], %[[LOOP_HEADER]] ], [ 100, %[[EARLY_EXIT_0]] ], [ 43, %[[LOOP_LATCH]] ], [ [[TMP9]], %[[VECTOR_EARLY_EXIT_0]] ], [ 100, %[[VECTOR_EARLY_EXIT_1]] ]
; CHECK-NEXT: ret i64 [[RETVAL]]
;
entry:
@@ -64,32 +95,63 @@ exit:
define i64 @two_early_exits_
diff erent_exits() {
; CHECK-LABEL: define i64 @two_early_exits_
diff erent_exits() {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT1:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[GEP_A]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ult <4 x i8> [[WIDE_LOAD]], splat (i8 34)
+; CHECK-NEXT: [[INDEX_NEXT1]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT1]], 64
+; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 false)
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: br label %[[EXIT2:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
-; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[GEP_A]], align 1
-; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 64, %[[SCALAR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV1]]
+; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[GEP_A1]], align 1
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV1]]
; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
-; CHECK-NEXT: br i1 [[CMP1]], label %[[EXIT1:.*]], label %[[EARLY_EXIT_0:.*]]
+; CHECK-NEXT: br i1 [[CMP1]], label %[[EXIT1]], label %[[EARLY_EXIT_0:.*]]
; CHECK: [[EARLY_EXIT_0]]:
; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i8 [[LD1]], 34
-; CHECK-NEXT: br i1 [[CMP2]], label %[[EXIT2:.*]], label %[[LOOP_LATCH]]
+; CHECK-NEXT: br i1 [[CMP2]], label %[[EXIT2]], label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP_HEADER]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP_HEADER]], label %[[EXIT:.*]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT1]]:
-; CHECK-NEXT: [[RET2:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ]
+; CHECK-NEXT: [[RET2:%.*]] = phi i64 [ [[IV1]], %[[LOOP_HEADER]] ], [ [[TMP9]], %[[VECTOR_EARLY_EXIT_0]] ]
; CHECK-NEXT: ret i64 [[RET2]]
; CHECK: [[EXIT2]]:
-; CHECK-NEXT: [[RET3:%.*]] = phi i64 [ 100, %[[EARLY_EXIT_0]] ]
+; CHECK-NEXT: [[RET3:%.*]] = phi i64 [ 100, %[[EARLY_EXIT_0]] ], [ 100, %[[VECTOR_EARLY_EXIT_1]] ]
; CHECK-NEXT: ret i64 [[RET3]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i64 43
@@ -200,32 +262,52 @@ exit:
define i64 @three_early_exits_same_exit() {
; CHECK-LABEL: define i64 @three_early_exits_same_exit() {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
-; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
-; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ 3, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX]]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[TMP0]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i8 [[LD_A]], 42
-; CHECK-NEXT: br i1 [[TMP8]], label %[[EXIT:.*]], label %[[EARLY_EXIT_0:.*]]
-; CHECK: [[EARLY_EXIT_0]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], splat (i8 42)
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[LD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
-; CHECK-NEXT: [[CMP2:%.*]] = icmp eq i8 [[LD_A]], [[LD_B]]
-; CHECK-NEXT: br i1 [[CMP2]], label %[[EXIT]], label %[[EARLY_EXIT_1:.*]]
-; CHECK: [[EARLY_EXIT_1]]:
-; CHECK-NEXT: [[CMP3:%.*]] = icmp ugt i8 [[LD_B]], 100
-; CHECK-NEXT: br i1 [[CMP3]], label %[[EXIT]], label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[OFFSET_IDX]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], 67
-; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt <4 x i8> [[WIDE_LOAD1]], splat (i8 100)
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> splat (i1 true), <4 x i1> [[TMP4]]
+; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[TMP6]]
+; CHECK-NEXT: [[CMP2:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
+; CHECK-NEXT: br i1 [[CMP2]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP1]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_CHECK_0:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK_0]]:
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP3]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[VECTOR_EARLY_EXIT_1:.*]], label %[[VECTOR_EARLY_EXIT_2:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_2]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 3, [[TMP12]]
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[OFFSET_IDX]], %[[LOOP_HEADER]] ], [ 100, %[[EARLY_EXIT_0]] ], [ 200, %[[EARLY_EXIT_1]] ], [ 43, %[[LOOP_LATCH]] ]
+; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ 200, %[[VECTOR_EARLY_EXIT_2]] ], [ 100, %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP13]], %[[VECTOR_EARLY_EXIT_0]] ], [ 43, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RETVAL]]
;
entry:
@@ -264,39 +346,63 @@ exit:
define i64 @four_early_exits_same_exit() {
; CHECK-LABEL: define i64 @four_early_exits_same_exit() {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P3:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P3]], i64 1024)
-; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
-; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ 3, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX]]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[TMP0]], align 1
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i8 [[LD_A]], 42
-; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[EARLY_EXIT_0:.*]]
-; CHECK: [[EARLY_EXIT_0]]:
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], splat (i8 42)
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[LD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
-; CHECK-NEXT: [[CMP2:%.*]] = icmp eq i8 [[LD_A]], [[LD_B]]
-; CHECK-NEXT: br i1 [[CMP2]], label %[[EXIT]], label %[[EARLY_EXIT_1:.*]]
-; CHECK: [[EARLY_EXIT_1]]:
-; CHECK-NEXT: [[GEP_C:%.*]] = getelementptr inbounds i8, ptr [[P3]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[LD_C:%.*]] = load i8, ptr [[GEP_C]], align 1
-; CHECK-NEXT: [[CMP3:%.*]] = icmp eq i8 [[LD_B]], [[LD_C]]
-; CHECK-NEXT: br i1 [[CMP3]], label %[[EXIT]], label %[[EARLY_EXIT_2:.*]]
-; CHECK: [[EARLY_EXIT_2]]:
-; CHECK-NEXT: [[CMP4:%.*]] = icmp ugt i8 [[LD_C]], 100
-; CHECK-NEXT: br i1 [[CMP4]], label %[[EXIT]], label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[OFFSET_IDX]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], 67
-; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P3]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ugt <4 x i8> [[WIDE_LOAD2]], splat (i8 100)
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP7]], <4 x i1> splat (i1 true), <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP8]], <4 x i1> splat (i1 true), <4 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP9]]
+; CHECK-NEXT: [[CMP2:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
+; CHECK-NEXT: br i1 [[CMP2]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP9]], i1 false)
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP1]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_CHECK_0:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK_0]]:
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP3]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[VECTOR_EARLY_EXIT_1:.*]], label %[[VECTOR_EARLY_EXIT_CHECK_1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK_1]]:
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP5]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[VECTOR_EARLY_EXIT_2:.*]], label %[[VECTOR_EARLY_EXIT_3:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_3]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VECTOR_EARLY_EXIT_2]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: [[TMP17:%.*]] = add i64 3, [[TMP16]]
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[OFFSET_IDX]], %[[LOOP_HEADER]] ], [ 100, %[[EARLY_EXIT_0]] ], [ 200, %[[EARLY_EXIT_1]] ], [ 300, %[[EARLY_EXIT_2]] ], [ 43, %[[LOOP_LATCH]] ]
+; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ 300, %[[VECTOR_EARLY_EXIT_3]] ], [ 200, %[[VECTOR_EARLY_EXIT_2]] ], [ 100, %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP17]], %[[VECTOR_EARLY_EXIT_0]] ], [ 43, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RETVAL]]
;
entry:
@@ -343,31 +449,50 @@ exit:
define i64 @two_early_exits_with_live_out_values() {
; CHECK-LABEL: define i64 @two_early_exits_with_live_out_values() {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
-; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
-; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ 3, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX]]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[LD_B:%.*]] = load i8, ptr [[TMP1]], align 1
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i8 [[LD_A]], [[LD_B]]
-; CHECK-NEXT: br i1 [[TMP7]], label %[[EXIT:.*]], label %[[EARLY_EXIT_0:.*]]
-; CHECK: [[EARLY_EXIT_0]]:
-; CHECK-NEXT: [[SUM:%.*]] = add i8 [[LD_A]], [[LD_B]]
-; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i8 [[SUM]], 34
-; CHECK-NEXT: br i1 [[CMP2]], label %[[EXIT]], label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[OFFSET_IDX]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], 67
-; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ult <4 x i8> [[TMP3]], splat (i8 34)
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> splat (i1 true), <4 x i1> [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
+; CHECK-NEXT: br i1 [[TMP7]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP2]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i8> [[TMP3]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 3, [[TMP11]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: [[TMP14:%.*]] = add i64 3, [[TMP13]]
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[OFFSET_IDX]], %[[LOOP_HEADER]] ], [ [[OFFSET_IDX]], %[[EARLY_EXIT_0]] ], [ 99, %[[LOOP_LATCH]] ]
-; CHECK-NEXT: [[RETVAL2:%.*]] = phi i8 [ 0, %[[LOOP_HEADER]] ], [ [[SUM]], %[[EARLY_EXIT_0]] ], [ 0, %[[LOOP_LATCH]] ]
+; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[TMP12]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP14]], %[[VECTOR_EARLY_EXIT_0]] ], [ 99, %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RETVAL2:%.*]] = phi i8 [ [[TMP10]], %[[VECTOR_EARLY_EXIT_1]] ], [ 0, %[[VECTOR_EARLY_EXIT_0]] ], [ 0, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[EXT:%.*]] = zext i8 [[RETVAL2]] to i64
; CHECK-NEXT: [[RET:%.*]] = add i64 [[RETVAL]], [[EXT]]
; CHECK-NEXT: ret i64 [[RET]]
@@ -408,29 +533,62 @@ exit:
define i64 @two_early_exits_negated_condition() {
; CHECK-LABEL: define i64 @two_early_exits_negated_condition() {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = add i64 3, [[INDEX]]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[GEP_A]], align 1
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP11]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ult <4 x i8> [[WIDE_LOAD]], splat (i8 34)
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 124
+; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 false)
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: [[TMP10:%.*]] = add i64 3, [[TMP9]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ 3, %[[ENTRY]] ]
-; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
-; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ 127, %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[GEP_A1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV1]]
+; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[GEP_A1]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV1]]
; CHECK-NEXT: [[LD_B:%.*]] = load i8, ptr [[TMP1]], align 1
; CHECK-NEXT: [[CMP1:%.*]] = icmp ne i8 [[LD_A]], [[LD_B]]
-; CHECK-NEXT: br i1 [[CMP1]], label %[[EARLY_EXIT_0:.*]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[CMP1]], label %[[EARLY_EXIT_0:.*]], label %[[EXIT]]
; CHECK: [[EARLY_EXIT_0]]:
; CHECK-NEXT: [[CMP2:%.*]] = icmp uge i8 [[LD_A]], 34
; CHECK-NEXT: br i1 [[CMP2]], label %[[LOOP_LATCH]], label %[[EXIT]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], 128
-; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ], [ 100, %[[EARLY_EXIT_0]] ], [ 43, %[[LOOP_LATCH]] ]
+; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[IV1]], %[[LOOP_HEADER]] ], [ 100, %[[EARLY_EXIT_0]] ], [ 43, %[[LOOP_LATCH]] ], [ [[TMP10]], %[[VECTOR_EARLY_EXIT_0]] ], [ 100, %[[VECTOR_EARLY_EXIT_1]] ]
; CHECK-NEXT: ret i64 [[RETVAL]]
;
entry:
@@ -466,39 +624,56 @@ exit:
; Three early exits to three
diff erent exit blocks.
define i64 @three_early_exits_three_exit_blocks() {
; CHECK-LABEL: define i64 @three_early_exits_three_exit_blocks() {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
-; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
-; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[OFFSET_IDX:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ 3, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX]]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT: [[LD_B:%.*]] = load i8, ptr [[TMP1]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i8 [[LD_A]], [[LD_B]]
-; CHECK-NEXT: br i1 [[TMP8]], label %[[EXIT1:.*]], label %[[EARLY_EXIT_0:.*]]
-; CHECK: [[EARLY_EXIT_0]]:
-; CHECK-NEXT: [[TMP10:%.*]] = icmp ult i8 [[LD_A]], 34
-; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT2:.*]], label %[[EARLY_EXIT_1:.*]]
-; CHECK: [[EARLY_EXIT_1]]:
-; CHECK-NEXT: [[TMP11:%.*]] = icmp ugt i8 [[LD_B]], 100
-; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT3:.*]], label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[OFFSET_IDX]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], 99
-; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT:.*]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ult <4 x i8> [[WIDE_LOAD]], splat (i8 34)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt <4 x i8> [[WIDE_LOAD1]], splat (i8 100)
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> splat (i1 true), <4 x i1> [[TMP4]]
+; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
+; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP2]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_CHECK_0:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK_0]]:
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP3]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[VECTOR_EARLY_EXIT_1:.*]], label %[[VECTOR_EARLY_EXIT_2:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_2]]:
+; CHECK-NEXT: br label %[[EXIT3:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: br label %[[EXIT2:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: [[RET3:%.*]] = add i64 3, [[TMP12]]
+; CHECK-NEXT: br label %[[EXIT1:.*]]
; CHECK: [[EXIT1]]:
-; CHECK-NEXT: [[RET4:%.*]] = phi i64 [ [[OFFSET_IDX]], %[[LOOP_HEADER]] ]
-; CHECK-NEXT: ret i64 [[RET4]]
+; CHECK-NEXT: ret i64 [[RET3]]
; CHECK: [[EXIT2]]:
-; CHECK-NEXT: [[RET2:%.*]] = phi i64 [ 100, %[[EARLY_EXIT_0]] ]
-; CHECK-NEXT: ret i64 [[RET2]]
+; CHECK-NEXT: ret i64 100
; CHECK: [[EXIT3]]:
-; CHECK-NEXT: [[RET3:%.*]] = phi i64 [ 200, %[[EARLY_EXIT_1]] ]
-; CHECK-NEXT: ret i64 [[RET3]]
+; CHECK-NEXT: ret i64 200
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i64 43
;
@@ -573,7 +748,7 @@ define i64 @two_early_exits_iv_live_out() {
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT]]:
@@ -616,32 +791,49 @@ exit:
define i64 @three_early_exits_iv_and_load_live_out() {
; CHECK-LABEL: define i64 @three_early_exits_iv_and_load_live_out() {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
-; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
-; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
-; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
-; CHECK-NEXT: [[LD_B:%.*]] = load i8, ptr [[TMP1]], align 1
-; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LD_A]], [[LD_B]]
-; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i8 [[LD_A]], 34
-; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[CMP1]], [[CMP2]]
-; CHECK-NEXT: br i1 [[TMP8]], label %[[EXIT:.*]], label %[[EARLY_EXIT_1:.*]]
-; CHECK: [[EARLY_EXIT_1]]:
-; CHECK-NEXT: [[CMP3:%.*]] = icmp ugt i8 [[LD_B]], 100
-; CHECK-NEXT: br i1 [[CMP3]], label %[[EXIT]], label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[INDEX]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 128
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ult <4 x i8> [[WIDE_LOAD]], splat (i8 34)
+; CHECK-NEXT: [[TMP4:%.*]] = or <4 x i1> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ugt <4 x i8> [[WIDE_LOAD1]], splat (i8 100)
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> splat (i1 true), <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP7]])
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP4]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP10]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i8> [[WIDE_LOAD1]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i8> [[WIDE_LOAD]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RETVAL_IV:%.*]] = phi i64 [ [[INDEX]], %[[LOOP_HEADER]] ], [ 128, %[[LOOP_LATCH]] ], [ [[INDEX]], %[[EARLY_EXIT_1]] ]
-; CHECK-NEXT: [[RETVAL_LD:%.*]] = phi i8 [ [[LD_A]], %[[LOOP_HEADER]] ], [ 0, %[[LOOP_LATCH]] ], [ [[LD_B]], %[[EARLY_EXIT_1]] ]
+; CHECK-NEXT: [[RETVAL_IV:%.*]] = phi i64 [ [[TMP12]], %[[VECTOR_EARLY_EXIT_1]] ], [ 128, %[[MIDDLE_BLOCK]] ], [ [[TMP14]], %[[VECTOR_EARLY_EXIT_0]] ]
+; CHECK-NEXT: [[RETVAL_LD:%.*]] = phi i8 [ [[TMP11]], %[[VECTOR_EARLY_EXIT_1]] ], [ 0, %[[MIDDLE_BLOCK]] ], [ [[TMP13]], %[[VECTOR_EARLY_EXIT_0]] ]
; CHECK-NEXT: [[EXT:%.*]] = zext i8 [[RETVAL_LD]] to i64
; CHECK-NEXT: [[RET:%.*]] = add i64 [[RETVAL_IV]], [[EXT]]
; CHECK-NEXT: ret i64 [[RET]]
@@ -763,30 +955,45 @@ exit.latch:
; Two early exits to same block, IV live-out with
diff erent incoming values.
define i64 @two_early_exits_iv_
diff _incoming() {
; CHECK-LABEL: define i64 @two_early_exits_iv_
diff _incoming() {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
-; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
-; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
-; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
-; CHECK-NEXT: [[LD_B:%.*]] = load i8, ptr [[TMP1]], align 1
-; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i8 [[LD_A]], [[LD_B]]
-; CHECK-NEXT: br i1 [[TMP14]], label %[[EXIT:.*]], label %[[EARLY_EXIT_0:.*]]
-; CHECK: [[EARLY_EXIT_0]]:
-; CHECK-NEXT: [[IV_PLUS1:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i8 [[LD_A]], 34
-; CHECK-NEXT: br i1 [[CMP2]], label %[[EXIT]], label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[INDEX]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 128
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ult <4 x i8> [[WIDE_LOAD]], splat (i8 34)
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
+; CHECK-NEXT: [[TMP14:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP14]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 false)
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[TMP9]], 1
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], %[[LOOP_HEADER]] ], [ [[IV_PLUS1]], %[[EARLY_EXIT_0]] ], [ 200, %[[LOOP_LATCH]] ]
+; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[TMP10]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP11]], %[[VECTOR_EARLY_EXIT_0]] ], [ 200, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RETVAL]]
;
entry:
@@ -822,35 +1029,58 @@ exit:
define { i64, i64 } @three_early_exits_multiple_live_outs() {
; CHECK-LABEL: define { i64, i64 } @three_early_exits_multiple_live_outs() {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
-; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
-; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
-; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
-; CHECK-NEXT: [[LD_B:%.*]] = load i8, ptr [[TMP1]], align 1
-; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i8 [[LD_A]], [[LD_B]]
-; CHECK-NEXT: br i1 [[TMP10]], label %[[EXIT:.*]], label %[[EARLY_EXIT_0:.*]]
-; CHECK: [[EARLY_EXIT_0]]:
-; CHECK-NEXT: [[SUM:%.*]] = add i8 [[LD_A]], [[LD_B]]
-; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i8 [[SUM]], 34
-; CHECK-NEXT: br i1 [[CMP2]], label %[[EXIT]], label %[[EARLY_EXIT_1:.*]]
-; CHECK: [[EARLY_EXIT_1]]:
-; CHECK-NEXT: [[DIFF:%.*]] = sub i8 [[LD_A]], [[LD_B]]
-; CHECK-NEXT: [[CMP3:%.*]] = icmp ugt i8 [[DIFF]], 100
-; CHECK-NEXT: br i1 [[CMP3]], label %[[EXIT]], label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[INDEX]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 128
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP4:%.*]] = icmp ult <4 x i8> [[TMP3]], splat (i8 34)
+; CHECK-NEXT: [[TMP5:%.*]] = sub <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ugt <4 x i8> [[TMP5]], splat (i8 100)
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> splat (i1 true), <4 x i1> [[TMP4]]
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP7]], <4 x i1> splat (i1 true), <4 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <4 x i1> [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP9]])
+; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[TMP10]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP8]], i1 false)
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP2]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_CHECK_0:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK_0]]:
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP4]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[VECTOR_EARLY_EXIT_1:.*]], label %[[VECTOR_EARLY_EXIT_2:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_2]]:
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i8> [[TMP5]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i8> [[TMP3]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: [[TMP17:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i8> [[WIDE_LOAD]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RETVAL_IV:%.*]] = phi i64 [ [[INDEX]], %[[LOOP_HEADER]] ], [ [[INDEX]], %[[EARLY_EXIT_0]] ], [ [[INDEX]], %[[EARLY_EXIT_1]] ], [ 128, %[[LOOP_LATCH]] ]
-; CHECK-NEXT: [[RETVAL_VAL:%.*]] = phi i8 [ [[LD_A]], %[[LOOP_HEADER]] ], [ [[SUM]], %[[EARLY_EXIT_0]] ], [ [[DIFF]], %[[EARLY_EXIT_1]] ], [ 0, %[[LOOP_LATCH]] ]
+; CHECK-NEXT: [[RETVAL_IV:%.*]] = phi i64 [ [[TMP15]], %[[VECTOR_EARLY_EXIT_2]] ], [ [[TMP17]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP19]], %[[VECTOR_EARLY_EXIT_0]] ], [ 128, %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RETVAL_VAL:%.*]] = phi i8 [ [[TMP14]], %[[VECTOR_EARLY_EXIT_2]] ], [ [[TMP16]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP18]], %[[VECTOR_EARLY_EXIT_0]] ], [ 0, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[EXT:%.*]] = zext i8 [[RETVAL_VAL]] to i64
; CHECK-NEXT: [[R1:%.*]] = insertvalue { i64, i64 } undef, i64 [[RETVAL_IV]], 0
; CHECK-NEXT: [[R2:%.*]] = insertvalue { i64, i64 } [[R1]], i64 [[EXT]], 1
@@ -899,29 +1129,44 @@ exit:
; Two early exits with second load only executed conditionally (after first exit check).
define i64 @two_early_exits_load_in_early_exit_block() {
; CHECK-LABEL: define i64 @two_early_exits_load_in_early_exit_block() {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
-; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
-; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
-; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i8 [[LD_A]], 42
-; CHECK-NEXT: br i1 [[TMP6]], label %[[EXIT:.*]], label %[[EARLY_EXIT_0:.*]]
-; CHECK: [[EARLY_EXIT_0]]:
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], splat (i8 42)
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
-; CHECK-NEXT: [[LD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
-; CHECK-NEXT: [[CMP2:%.*]] = icmp eq i8 [[LD_A]], [[LD_B]]
-; CHECK-NEXT: br i1 [[CMP2]], label %[[EXIT]], label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 128
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
+; CHECK-NEXT: [[CMP2:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[CMP2]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 false)
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP1]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ], [ [[IV]], %[[EARLY_EXIT_0]] ], [ 128, %[[LOOP_LATCH]] ]
+; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[TMP9]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP10]], %[[VECTOR_EARLY_EXIT_0]] ], [ 128, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RETVAL]]
;
entry:
@@ -956,36 +1201,56 @@ exit:
define i64 @three_early_exits_loads_in_
diff erent_blocks() {
; CHECK-LABEL: define i64 @three_early_exits_loads_in_
diff erent_blocks() {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P3:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P3]], i64 1024)
-; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
-; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
-; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
-; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i8 [[LD_A]], 42
-; CHECK-NEXT: br i1 [[TMP9]], label %[[EXIT:.*]], label %[[EARLY_EXIT_0:.*]]
-; CHECK: [[EARLY_EXIT_0]]:
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], splat (i8 42)
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
-; CHECK-NEXT: [[LD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
-; CHECK-NEXT: [[CMP2:%.*]] = icmp eq i8 [[LD_A]], [[LD_B]]
-; CHECK-NEXT: br i1 [[CMP2]], label %[[EXIT]], label %[[EARLY_EXIT_1:.*]]
-; CHECK: [[EARLY_EXIT_1]]:
-; CHECK-NEXT: [[GEP_C:%.*]] = getelementptr inbounds i8, ptr [[P3]], i64 [[IV]]
-; CHECK-NEXT: [[LD_C:%.*]] = load i8, ptr [[GEP_C]], align 1
-; CHECK-NEXT: [[CMP3:%.*]] = icmp eq i8 [[LD_B]], [[LD_C]]
-; CHECK-NEXT: br i1 [[CMP3]], label %[[EXIT]], label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 128
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P3]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD2]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> splat (i1 true), <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP8:%.*]] = freeze <4 x i1> [[TMP7]]
+; CHECK-NEXT: [[CMP2:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[CMP2]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP7]], i1 false)
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP1]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP11]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_CHECK_0:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK_0]]:
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP3]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[VECTOR_EARLY_EXIT_1:.*]], label %[[VECTOR_EARLY_EXIT_2:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_2]]:
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ], [ [[IV]], %[[EARLY_EXIT_0]] ], [ [[IV]], %[[EARLY_EXIT_1]] ], [ 128, %[[LOOP_LATCH]] ]
+; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[TMP13]], %[[VECTOR_EARLY_EXIT_2]] ], [ [[TMP14]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP15]], %[[VECTOR_EARLY_EXIT_0]] ], [ 128, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RETVAL]]
;
entry:
@@ -1028,7 +1293,7 @@ exit:
define { i64, i8 } @four_early_exits_with_conditional_loads() {
; CHECK-LABEL: define { i64, i8 } @four_early_exits_with_conditional_loads() {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P3:%.*]] = alloca [1024 x i8], align 1
@@ -1037,35 +1302,64 @@ define { i64, i8 } @four_early_exits_with_conditional_loads() {
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P3]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P4]], i64 1024)
-; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
-; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
-; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
-; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i8 [[LD_A]], 10
-; CHECK-NEXT: br i1 [[TMP12]], label %[[EXIT:.*]], label %[[EARLY_EXIT_0:.*]]
-; CHECK: [[EARLY_EXIT_0]]:
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], splat (i8 10)
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
-; CHECK-NEXT: [[LD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
-; CHECK-NEXT: [[CMP2:%.*]] = icmp eq i8 [[LD_B]], 20
-; CHECK-NEXT: br i1 [[CMP2]], label %[[EXIT]], label %[[EARLY_EXIT_1:.*]]
-; CHECK: [[EARLY_EXIT_1]]:
-; CHECK-NEXT: [[GEP_C:%.*]] = getelementptr inbounds i8, ptr [[P3]], i64 [[IV]]
-; CHECK-NEXT: [[LD_C:%.*]] = load i8, ptr [[GEP_C]], align 1
-; CHECK-NEXT: [[CMP3:%.*]] = icmp eq i8 [[LD_C]], 30
-; CHECK-NEXT: br i1 [[CMP3]], label %[[EXIT]], label %[[EARLY_EXIT_2:.*]]
-; CHECK: [[EARLY_EXIT_2]]:
-; CHECK-NEXT: [[GEP_D:%.*]] = getelementptr inbounds i8, ptr [[P4]], i64 [[IV]]
-; CHECK-NEXT: [[LD_D:%.*]] = load i8, ptr [[GEP_D]], align 1
-; CHECK-NEXT: [[CMP4:%.*]] = icmp eq i8 [[LD_D]], 40
-; CHECK-NEXT: br i1 [[CMP4]], label %[[EXIT]], label %[[LOOP_LATCH]]
-; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 128
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]]
+; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD1]], splat (i8 20)
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P3]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD2]], splat (i8 30)
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[P4]], i64 [[IV]]
+; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP6]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD3]], splat (i8 40)
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP8]], <4 x i1> splat (i1 true), <4 x i1> [[TMP5]]
+; CHECK-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> splat (i1 true), <4 x i1> [[TMP7]]
+; CHECK-NEXT: [[TMP11:%.*]] = freeze <4 x i1> [[TMP10]]
+; CHECK-NEXT: [[CMP2:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP11]])
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT: br i1 [[CMP2]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP10]], i1 false)
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP1]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_CHECK_0:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK_0]]:
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP3]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP15]], label %[[VECTOR_EARLY_EXIT_1:.*]], label %[[VECTOR_EARLY_EXIT_CHECK_1:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_CHECK_1]]:
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP5]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br i1 [[TMP16]], label %[[VECTOR_EARLY_EXIT_2:.*]], label %[[VECTOR_EARLY_EXIT_3:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT_3]]:
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i8> [[WIDE_LOAD3]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VECTOR_EARLY_EXIT_2]]:
+; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i8> [[WIDE_LOAD2]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: [[TMP20:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VECTOR_EARLY_EXIT_1]]:
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i8> [[WIDE_LOAD1]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[VECTOR_EARLY_EXIT_0]]:
+; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i8> [[WIDE_LOAD]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RETVAL_IV:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ], [ [[IV]], %[[EARLY_EXIT_0]] ], [ [[IV]], %[[EARLY_EXIT_1]] ], [ [[IV]], %[[EARLY_EXIT_2]] ], [ 128, %[[LOOP_LATCH]] ]
-; CHECK-NEXT: [[RETVAL_VAL:%.*]] = phi i8 [ [[LD_A]], %[[LOOP_HEADER]] ], [ [[LD_B]], %[[EARLY_EXIT_0]] ], [ [[LD_C]], %[[EARLY_EXIT_1]] ], [ [[LD_D]], %[[EARLY_EXIT_2]] ], [ 0, %[[LOOP_LATCH]] ]
+; CHECK-NEXT: [[RETVAL_IV:%.*]] = phi i64 [ [[TMP18]], %[[VECTOR_EARLY_EXIT_3]] ], [ [[TMP20]], %[[VECTOR_EARLY_EXIT_2]] ], [ [[TMP22]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP24]], %[[VECTOR_EARLY_EXIT_0]] ], [ 128, %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[RETVAL_VAL:%.*]] = phi i8 [ [[TMP17]], %[[VECTOR_EARLY_EXIT_3]] ], [ [[TMP19]], %[[VECTOR_EARLY_EXIT_2]] ], [ [[TMP21]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP23]], %[[VECTOR_EARLY_EXIT_0]] ], [ 0, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[R1:%.*]] = insertvalue { i64, i8 } undef, i64 [[RETVAL_IV]], 0
; CHECK-NEXT: [[R2:%.*]] = insertvalue { i64, i8 } [[R1]], i8 [[RETVAL_VAL]], 1
; CHECK-NEXT: ret { i64, i8 } [[R2]]
diff --git a/llvm/test/Transforms/LoopVectorize/unsupported_early_exit.ll b/llvm/test/Transforms/LoopVectorize/unsupported_early_exit.ll
index 80fbb6e7a49ca..614c98c6b8016 100644
--- a/llvm/test/Transforms/LoopVectorize/unsupported_early_exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/unsupported_early_exit.ll
@@ -57,65 +57,6 @@ loop.end:
}
-; We don't currently support multiple early exits.
-define i64 @multiple_uncountable_exits() {
-; CHECK-LABEL: define i64 @multiple_uncountable_exits() {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
-; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
-; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
-; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
-; CHECK-NEXT: br label [[SEARCH1:%.*]]
-; CHECK: search1:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[ENTRY:%.*]] ]
-; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
-; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
-; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
-; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
-; CHECK-NEXT: br i1 [[CMP1]], label [[LOOP_END:%.*]], label [[SEARCH2:%.*]]
-; CHECK: search2:
-; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i8 [[LD1]], 34
-; CHECK-NEXT: br i1 [[CMP2]], label [[LOOP_END]], label [[LOOP_INC]]
-; CHECK: loop.inc:
-; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[SEARCH1]], label [[LOOP_END]]
-; CHECK: loop.end:
-; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[SEARCH1]] ], [ 100, [[SEARCH2]] ], [ 43, [[LOOP_INC]] ]
-; CHECK-NEXT: ret i64 [[RETVAL]]
-;
-entry:
- %p1 = alloca [1024 x i8]
- %p2 = alloca [1024 x i8]
- call void @init_mem(ptr %p1, i64 1024)
- call void @init_mem(ptr %p2, i64 1024)
- br label %search1
-
-search1:
- %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
- %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
- %ld1 = load i8, ptr %arrayidx, align 1
- %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
- %ld2 = load i8, ptr %arrayidx1, align 1
- %cmp1 = icmp eq i8 %ld1, %ld2
- br i1 %cmp1, label %loop.end, label %search2
-
-search2:
- %cmp2 = icmp ult i8 %ld1, 34
- br i1 %cmp2, label %loop.end, label %loop.inc
-
-loop.inc:
- %index.next = add i64 %index, 1
- %exitcond = icmp ne i64 %index.next, 67
- br i1 %exitcond, label %search1, label %loop.end
-
-loop.end:
- %retval = phi i64 [ %index, %search1 ], [ 100, %search2 ], [ 43, %loop.inc ]
- ret i64 %retval
-}
-
-
define i64 @uncountable_exit_infinite_loop() {
; CHECK-LABEL: define i64 @uncountable_exit_infinite_loop() {
; CHECK-NEXT: entry:
@@ -709,6 +650,74 @@ loop.end:
ret i64 %retval
}
+; Two early exits with load (not known to be dereferenceable) in a non-exiting middle block between them. The load is only executed if the first early exit is not taken, so it needs predication. This should not be vectorized.
+define i64 @multi_exit_load_in_nonexiting_block(ptr %src) {
+; CHECK-LABEL: define i64 @multi_exit_load_in_nonexiting_block(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
+; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
+; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
+; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]
+; CHECK: loop.header:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
+; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[GEP1]], align 1
+; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LD1]], 42
+; CHECK-NEXT: br i1 [[CMP1]], label [[LOOP_END:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[LD_OUT:%.*]] = load i64, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT: br label [[EARLY_EXIT_2:%.*]]
+; CHECK: early.exit.2:
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
+; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[GEP2]], align 1
+; CHECK-NEXT: [[CMP2:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
+; CHECK-NEXT: br i1 [[CMP2]], label [[LOOP_END]], label [[LOOP_LATCH]]
+; CHECK: loop.latch:
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP_HEADER]], label [[LOOP_END]]
+; CHECK: loop.end:
+; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP_HEADER]] ], [ [[LD_OUT]], [[EARLY_EXIT_2]] ], [ 67, [[LOOP_LATCH]] ]
+; CHECK-NEXT: ret i64 [[RETVAL]]
+;
+entry:
+ %p1 = alloca [1024 x i8]
+ %p2 = alloca [1024 x i8]
+ call void @init_mem(ptr %p1, i64 1024)
+ call void @init_mem(ptr %p2, i64 1024)
+ br label %loop.header
+
+loop.header:
+ %index = phi i64 [ %index.next, %loop.latch ], [ 0, %entry ]
+ %gep1 = getelementptr inbounds i8, ptr %p1, i64 %index
+ %ld1 = load i8, ptr %gep1, align 1
+ %cmp1 = icmp eq i8 %ld1, 42
+ br i1 %cmp1, label %loop.end, label %middle.block
+
+middle.block:
+ %gep.src = getelementptr inbounds i64, ptr %src, i64 %index
+ %ld.out = load i64, ptr %gep.src
+ br label %early.exit.2
+
+early.exit.2:
+ ; Second early exit block
+ %gep2 = getelementptr inbounds i8, ptr %p2, i64 %index
+ %ld2 = load i8, ptr %gep2, align 1
+ %cmp2 = icmp eq i8 %ld1, %ld2
+ br i1 %cmp2, label %loop.end, label %loop.latch
+
+loop.latch:
+ %index.next = add i64 %index, 1
+ %exitcond = icmp ne i64 %index.next, 67
+ br i1 %exitcond, label %loop.header, label %loop.end
+
+loop.end:
+ %retval = phi i64 [ %index, %loop.header ], [ %ld.out, %early.exit.2 ], [ 67, %loop.latch ]
+ ret i64 %retval
+}
declare i32 @foo(i32) readonly
declare <vscale x 4 x i32> @foo_vec(<vscale x 4 x i32>)
More information about the llvm-commits
mailing list