[llvm] [LV] Use speculative load intrinsics for early-exit vectorization (PR #180039)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun Sep 27 15:00:25 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/180039
>From 3068bf7441864c83591f1b734bc4496d84b158b7 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 10 Mar 2026 15:04:36 +0000
Subject: [PATCH 1/2] [VPlan] Handle broadcast in getSCEVExprForVPValue (NFC)
A broadcast just replicates a scalar value, so its SCEV expression is
the same as its operand's. This allows getSCEVExprForVPValue to see
through broadcasts when computing SCEV expressions for VPValues.
---
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 3 +++
1 file changed, 3 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index a56689848efdf8..fbc657ad16854d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -165,6 +165,9 @@ const SCEV *vputils::getSCEVExprForVPValue(const VPValue *V,
};
VPValue *LHSVal, *RHSVal;
+ // Broadcast just replicates a scalar, so the SCEV is the same as its operand.
+ if (match(V, m_Broadcast(m_VPValue(LHSVal))))
+ return getSCEVExprForVPValue(LHSVal, PSE, L);
if (match(V, m_Add(m_VPValue(LHSVal), m_VPValue(RHSVal))))
return CreateSCEV({LHSVal, RHSVal}, [&](ArrayRef<SCEVUse> Ops) {
return SE.getAddExpr(Ops[0], Ops[1], SCEV::FlagNone, 0);
>From 8490fd17f6fa7e655822312d5bb030df2ab9a454 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sat, 21 Mar 2026 21:49:41 +0000
Subject: [PATCH 2/2] [LV] Use speculative load intrinsics for early-exit
vectorization
Add initial support for vectorizing loops using llvm.speculative.load
when loads are not known dereferenceable. The intrinsic was added in
https://github.com/llvm/llvm-project/pull/179642.
LV has to generate an oracle function, which returns the number of
elements the original scalar loop would have read, starting at the
current index of the vector loop.
To do so, introduce a new VPSpeculativeLoadRecipe, which models the
speculative load and also contains a VPlan that is used to generate the
oracle function during execute.
The oracle VPlan is the based on the original scalar VPlan. It replays
the scalar loop, starting at the current index of the vector loop
(passed to the oracle function as argument, same as other live-ins like
base pointers). The exits are updated to return the index of the IV that
exits the loop.
A new LiveIn VPInstruction has been added to model the live-in arguments
to the oracle; at the point where the oracle plan is created, no IR
values exist for them yet. During execute, they will be replaced by the
created function arguments.
The initial implementation limits the supported loops to cases where all
speculative loads share the same element type, access are guaranteed
consecutive and do not have multiple early exits, to keep the
implementation simple.
simplify
---
llvm/lib/Analysis/VectorUtils.cpp | 3 +
.../Vectorize/LoopVectorizationPlanner.h | 6 +
.../Transforms/Vectorize/LoopVectorize.cpp | 26 +-
llvm/lib/Transforms/Vectorize/VPlan.cpp | 34 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 38 +-
.../Vectorize/VPlanConstruction.cpp | 245 +++++-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 89 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 11 +-
.../Transforms/Vectorize/VPlanTransforms.h | 22 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 11 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 4 +
.../Transforms/Vectorize/VPlanVerifier.cpp | 20 +
.../early-exit-with-speculative-load-basic.ll | 518 +++++++++++-
...ly-exit-with-speculative-load-forced-vf.ll | 88 ++
...it-with-speculative-load-irregular-type.ll | 97 +++
...y-exit-with-speculative-load-load-types.ll | 232 +++++-
...early-exit-with-speculative-load-safety.ll | 454 +++++++++-
...rly-exit-with-speculative-load-scalable.ll | 71 +-
.../LoopVectorize/AArch64/early_exit_cost.ll | 121 ++-
...early-exit-with-speculative-load-oracle.ll | 786 +++++++++++++++++-
.../VPlan/vplan-print-before-after-all.ll | 1 +
.../LoopVectorize/unsupported_early_exit.ll | 1 +
.../Vectorize/VPlanVerifierTest.cpp | 62 ++
23 files changed, 2800 insertions(+), 140 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-forced-vf.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-irregular-type.ll
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 1c105ebb772b35..7c620c2a9a5775 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -158,6 +158,8 @@ bool llvm::isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID,
case Intrinsic::powi:
case Intrinsic::vector_extract:
return (ScalarOpdIdx == 1);
+ case Intrinsic::speculative_load:
+ return true;
case Intrinsic::smul_fix:
case Intrinsic::smul_fix_sat:
case Intrinsic::umul_fix:
@@ -196,6 +198,7 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg(
case Intrinsic::scmp:
case Intrinsic::vector_extract:
case Intrinsic::loop_dependence_war_mask:
+ case Intrinsic::speculative_load:
return OpdIdx == -1 || OpdIdx == 0;
case Intrinsic::modf:
case Intrinsic::sincos:
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 3c07a6e1596564..ebc854ad3368c9 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -489,6 +489,12 @@ template <typename InserterTy> class VPBuilderBase : public InserterTy {
return createNaryOp(Instruction::Freeze, Op, DL, Name);
}
+ VPInstruction *createLiveIn(Type *ResultTy, const Twine &Name = "") {
+ return tryInsertInstruction(new VPInstruction(VPInstruction::LiveIn, {}, {},
+ {}, DebugLoc::getUnknown(),
+ Name, ResultTy));
+ }
+
VPWidenCastRecipe *createWidenCast(Instruction::CastOps Opcode, VPValue *Op,
Type *ResultTy) {
assert(Op->getScalarType() != ResultTy &&
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e934fec3663314..69e4cc4abd01de 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -5645,7 +5645,7 @@ LoopVectorizationPlanner::computeBestVF() {
return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan};
}
- if (hasPlanWithVF(UserVF) && hasForcedEpilogueVF() && VPlans.size() == 2) {
+ if (FirstPlan.hasVF(UserVF) && hasForcedEpilogueVF() && VPlans.size() == 2) {
assert(VPlans[0]->getSingleVF() == UserVF &&
"expected second plan to be for the forced UserVF");
assert(VPlans[1]->getSingleVF() == EpilogueVectorizationForceVF &&
@@ -5786,6 +5786,8 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
RUN_VPLAN_PASS(VPlanTransforms::replicateByVF, BestVPlan, BestVF);
bool HasBranchWeights =
hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator());
+ RUN_VPLAN_PASS(VPlanTransforms::attachSpeculativeLoadChecks, BestVPlan,
+ BestVF, PSE, OrigLoop, HasBranchWeights);
if (HasBranchWeights) {
std::optional<unsigned> VScale = Config.getVScaleForTuning();
RUN_VPLAN_PASS(VPlanTransforms::addBranchWeightToMiddleTerminator,
@@ -6569,6 +6571,19 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
return Plan;
}
+ if (auto *Oracle = vputils::findSpeculativeLoadOracle(*Plan)) {
+ auto IsLegal = [&](ElementCount VF) {
+ return all_of(Oracle->users(), [&](VPUser *U) {
+ auto *R = cast<VPWidenIntrinsicRecipe>(U);
+ return TTI.isLegalSpeculativeLoad(
+ toVectorTy(R->getScalarType(), VF),
+ R->getOperand(0)->getScalarType()->getPointerAddressSpace());
+ });
+ };
+ if (!LoopVectorizationPlanner::getDecisionAndClampRange(IsLegal, Range))
+ return nullptr;
+ }
+
using namespace llvm::VPlanPatternMatch;
SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
@@ -8044,6 +8059,15 @@ bool LoopVectorizePass::processLoop(Loop *L) {
IC = 1;
}
+ // FIXME: Enable interleaving for plans with speculative loads.
+ if (InterleaveLoop && vputils::findSpeculativeLoadOracle(*BestPlanPtr)) {
+ LLVM_DEBUG(dbgs() << "LV: Not interleaving loop with speculative loads.\n");
+ IntDiagMsg = {"SpeculativeLoadPreventsInterleaving",
+ "Unable to interleave loop using speculative loads."};
+ InterleaveLoop = false;
+ IC = 1;
+ }
+
// Emit diagnostic messages, if any.
if (!VectorizeLoop && !InterleaveLoop) {
// Do not vectorize or interleaving the loop.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 3cdad5161321a4..58924055368f80 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -578,7 +578,7 @@ const VPRegionBlock *VPBasicBlock::getEnclosingLoopRegion() const {
return getEnclosingLoopRegionForRegion(getParent());
}
-static bool hasConditionalTerminator(const VPBasicBlock *VPBB) {
+static bool hasTerminator(const VPBasicBlock *VPBB) {
if (VPBB->empty()) {
assert(
VPBB->getNumSuccessors() < 2 &&
@@ -587,6 +587,8 @@ static bool hasConditionalTerminator(const VPBasicBlock *VPBB) {
}
const VPRecipeBase *R = &VPBB->back();
+ if (match(R, m_VPInstruction<Instruction::Ret>()))
+ return true;
[[maybe_unused]] bool IsSwitch =
isa<VPInstruction>(R) &&
cast<VPInstruction>(R)->getOpcode() == Instruction::Switch;
@@ -617,13 +619,13 @@ static bool hasConditionalTerminator(const VPBasicBlock *VPBB) {
}
VPRecipeBase *VPBasicBlock::getTerminator() {
- if (hasConditionalTerminator(this))
+ if (hasTerminator(this))
return &back();
return nullptr;
}
const VPRecipeBase *VPBasicBlock::getTerminator() const {
- if (hasConditionalTerminator(this))
+ if (hasTerminator(this))
return &back();
return nullptr;
}
@@ -1072,41 +1074,42 @@ bool VPlan::isOuterLoop() const {
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-void VPlan::printLiveIns(raw_ostream &O) const {
+void VPlan::printLiveIns(raw_ostream &O, const Twine &Indent) const {
VPSlotTracker SlotTracker(this);
if (!VF.user_empty()) {
- O << "\nLive-in ";
+ O << '\n' << Indent << "Live-in ";
VF.printAsOperand(O, SlotTracker);
O << " = VF";
}
if (!UF.user_empty()) {
- O << "\nLive-in ";
+ O << '\n' << Indent << "Live-in ";
UF.printAsOperand(O, SlotTracker);
O << " = UF";
}
if (!VFxUF.user_empty()) {
- O << "\nLive-in ";
+ O << '\n' << Indent << "Live-in ";
VFxUF.printAsOperand(O, SlotTracker);
O << " = VF * UF";
}
if (!VectorTripCount.user_empty()) {
- O << "\nLive-in ";
+ O << '\n' << Indent << "Live-in ";
VectorTripCount.printAsOperand(O, SlotTracker);
O << " = vector-trip-count";
}
if (BackedgeTakenCount && !BackedgeTakenCount->user_empty()) {
- O << "\nLive-in ";
+ O << '\n' << Indent << "Live-in ";
BackedgeTakenCount->printAsOperand(O, SlotTracker);
O << " = backedge-taken count";
}
O << "\n";
if (TripCount && !TripCount->user_empty()) {
+ O << Indent;
if (isa<VPIRValue>(TripCount))
O << "Live-in ";
TripCount->printAsOperand(O, SlotTracker);
@@ -1116,21 +1119,18 @@ void VPlan::printLiveIns(raw_ostream &O) const {
}
LLVM_DUMP_METHOD
-void VPlan::print(raw_ostream &O) const {
+void VPlan::print(raw_ostream &O, const Twine &Indent) const {
VPSlotTracker SlotTracker(this);
-
- O << "VPlan '" << getName() << "' {";
-
- printLiveIns(O);
+ O << Indent << "VPlan '" << getName() << "' {";
+ printLiveIns(O, Indent);
ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<const VPBlockBase *>>
RPOT(getEntry());
for (const VPBlockBase *Block : RPOT) {
O << '\n';
- Block->print(O, "", SlotTracker);
+ Block->print(O, Indent, SlotTracker);
}
-
- O << "}\n";
+ O << Indent << "}\n";
}
std::string VPlan::getName() const {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 746c0231f6fcf3..1718be757b8daa 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -428,6 +428,7 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
VPReductionSC,
VPReplicateSC,
VPScalarIVStepsSC,
+ VPSpeculativeLoadOracleSC,
VPVectorPointerSC,
VPVectorEndPointerSC,
VPWidenCallSC,
@@ -632,6 +633,7 @@ class LLVM_ABI_FOR_TEST VPSingleDefRecipe : public VPRecipeBase,
case VPRecipeBase::VPReductionSC:
case VPRecipeBase::VPReplicateSC:
case VPRecipeBase::VPScalarIVStepsSC:
+ case VPRecipeBase::VPSpeculativeLoadOracleSC:
case VPRecipeBase::VPVectorPointerSC:
case VPRecipeBase::VPVectorEndPointerSC:
case VPRecipeBase::VPWidenCallSC:
@@ -1315,6 +1317,8 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
// Represents the incoming loop-invariant alias-mask. All memory accesses
// in the loop must stay within the active lanes.
IncomingAliasMask,
+ // Yields a distinct scalar input, bound by its position in the entry block.
+ LiveIn,
// Increment the canonical IV separately for each unrolled part.
CanonicalIVIncrementForPart,
// Abstract instruction that compares two values and branches. This is
@@ -3532,6 +3536,36 @@ class LLVM_ABI_FOR_TEST VPBranchOnMaskRecipe : public VPRecipeBase,
}
};
+/// Defines the oracle function for a @llvm.speculative.load intrinsic call. It
+/// contains a nested VPlan which is used to generate the oracle function,
+/// returning the number of bytes read by the intrinsic. The entry block holds
+/// LiveIn recipes in argument order. The oracle executes with VF=1; its lane
+/// limit is passed as an ordinary argument.
+class VPSpeculativeLoadOracleRecipe : public VPSingleDefRecipe {
+ std::unique_ptr<VPlan> OraclePlan;
+
+public:
+ explicit VPSpeculativeLoadOracleRecipe(std::unique_ptr<VPlan> Oracle);
+
+ VP_CLASSOF_IMPL(VPRecipeBase::VPSpeculativeLoadOracleSC)
+
+ VPSpeculativeLoadOracleRecipe *clone() override;
+ void execute(VPTransformState &State) override;
+
+ InstructionCost computeCost(ElementCount, VPCostContext &) const override {
+ return 0;
+ }
+
+ /// Returns the plan the oracle function is generated from.
+ const VPlan &getOraclePlan() const { return *OraclePlan; }
+
+protected:
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ void printRecipe(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &Tracker) const override;
+#endif
+};
+
/// A recipe to combine multiple recipes into a single 'expression' recipe,
/// which should be considered a single entity for cost-modeling and transforms.
/// The recipe needs to be 'decomposed', i.e. replaced by its individual
@@ -5160,10 +5194,10 @@ class VPlan {
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the live-ins of this VPlan to \p O.
- void printLiveIns(raw_ostream &O) const;
+ void printLiveIns(raw_ostream &O, const Twine &Indent = "") const;
/// Print this VPlan to \p O.
- LLVM_ABI_FOR_TEST void print(raw_ostream &O) const;
+ LLVM_ABI_FOR_TEST void print(raw_ostream &O, const Twine &Indent = "") const;
/// Print this VPlan in DOT format to \p O.
LLVM_ABI_FOR_TEST void printDOT(raw_ostream &O) const;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 6c5f688835c15d..e9ef3379c549b0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -32,6 +32,7 @@
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/MDBuilder.h"
+#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/LoopVersioning.h"
@@ -43,6 +44,13 @@ using namespace llvm;
using namespace LoopVectorizationUtils;
using namespace VPlanPatternMatch;
+static cl::opt<bool> EnableEarlyExitVectorizationWithSpeculativeLoads(
+ "enable-early-exit-vectorization-with-speculative-loads", cl::init(false),
+ cl::Hidden,
+ cl::desc("Enable vectorization of read-only early exit loops with "
+ "uncountable exits and loads not known to be dereferenceable, "
+ "using @llvm.speculative.load"));
+
namespace {
// Class that is used to build the plain CFG for the incoming IR.
class PlainCFGBuilder {
@@ -1229,14 +1237,130 @@ void VPlanTransforms::createInLoopReductionRecipes(VPlan &Plan,
R->eraseFromParent();
}
-bool VPlanTransforms::areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB,
- Loop *TheLoop,
- PredicatedScalarEvolution &PSE,
- DominatorTree &DT,
- AssumptionCache *AC) {
+/// Clone the loop of plain CFG \p Plan into an oracle plan: a scalar loop with
+/// trip count VF replaying the exit conditions, returning safe lanes *
+/// \p AccessSize bytes. Returns the oracle and the values to pass to it.
+static std::pair<VPSpeculativeLoadOracleRecipe *, SmallVector<VPValue *>>
+buildOraclePlan(VPlan &Plan, uint64_t AccessSize) {
+ Type *IVTy = Plan.getIndexType();
+ auto OraclePlan =
+ std::make_unique<VPlan>(Plan.getScalarHeader()->getIRBasicBlock(), IVTy);
+ OraclePlan->setName("Speculative-load oracle");
+ OraclePlan->addVF(ElementCount::getFixed(1));
+ VPBasicBlock *NewEntry = OraclePlan->getEntry();
+ NewEntry->setName("oracle.entry");
+ auto *ExitVPBB = OraclePlan->createVPBasicBlock("oracle.exit");
+ auto [OrigHeader, OrigLatch] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
+ DenseMap<VPBlockBase *, VPBasicBlock *> Old2NewVPBBs = {
+ {OrigHeader->getPredecessors()[0], NewEntry}};
+ auto LoopBody = vp_rpo_plain_cfg_loop_body(OrigHeader);
+ for (VPBasicBlock *VPBB : LoopBody)
+ Old2NewVPBBs[VPBB] = OraclePlan->createVPBasicBlock(VPBB->getName());
+ VPBasicBlock *HeaderVPBB = Old2NewVPBBs[OrigHeader];
+ NewEntry->setSuccessors({HeaderVPBB});
+
+ VPValue *One = OraclePlan->getConstantInt(IVTy, 1);
+ VPBuilder Builder(HeaderVPBB);
+ auto *ScalarIVPHI = Builder.createScalarPhi({OraclePlan->getZero(IVTy)},
+ DebugLoc::getUnknown(), "index");
+
+ // Clone the loop blocks, redirecting all exits to ExitVPBB. Replace wide
+ // inductions with scalar equivalents starting at their current value, passed
+ // as argument to the oracle, and execute loads and GEPs as single scalars.
+ VPBuilder EntryBuilder(NewEntry);
+ DenseMap<VPValue *, VPValue *> OracleArgToValue;
+ auto CreateArg = [&](VPValue *V, const Twine &Name = "") -> VPValue * {
+ VPValue *Arg = EntryBuilder.createLiveIn(V->getScalarType(), Name);
+ OracleArgToValue[Arg] = V;
+ return Arg;
+ };
+ DenseMap<VPValue *, VPValue *> Old2NewVPValues;
+ for (VPIRValue *LiveIn : Plan.getLiveIns()) {
+ Value *V = LiveIn->getValue();
+ Old2NewVPValues[LiveIn] = isa<ConstantData>(V)
+ ? OraclePlan->getOrAddLiveIn(LiveIn)
+ : CreateArg(LiveIn, V->getName());
+ }
+ auto MapVPBB = [&](VPBlockBase *VPBB) -> VPBlockBase * {
+ return Old2NewVPBBs.lookup_or(VPBB, ExitVPBB);
+ };
+ for (VPBasicBlock *VPBB : LoopBody) {
+ VPBasicBlock *NewVPBB = Old2NewVPBBs[VPBB];
+ NewVPBB->setPredecessors(map_to_vector(VPBB->getPredecessors(), MapVPBB));
+ NewVPBB->setSuccessors(map_to_vector(VPBB->getSuccessors(), MapVPBB));
+ if (is_contained(NewVPBB->getSuccessors(), ExitVPBB))
+ ExitVPBB->getPredecessors().push_back(NewVPBB);
+ for (VPRecipeBase &R : *VPBB) {
+ if (auto *WideIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R)) {
+ Old2NewVPValues[WideIV] = Builder.createDerivedIV(
+ InductionDescriptor::IK_IntInduction, /*FPBinOp=*/nullptr,
+ CreateArg(WideIV, "iv.start"), ScalarIVPHI,
+ Old2NewVPValues.lookup(WideIV->getStepValue()));
+ continue;
+ }
+ auto *VPI = cast<VPInstruction>(&R);
+ VPSingleDefRecipe *NewR =
+ is_contained({Instruction::Load, Instruction::GetElementPtr},
+ VPI->getOpcode())
+ ? VPBuilder::createSingleScalarOp(
+ VPI->getOpcode(), VPI->operands(), /*Mask=*/nullptr, *VPI,
+ *VPI, VPI->getDebugLoc(), VPI->getUnderlyingInstr())
+ : VPI->clone();
+ NewVPBB->appendRecipe(NewR);
+ for (auto [Idx, Op] : enumerate(VPI->operands()))
+ NewR->setOperand(Idx, Old2NewVPValues.lookup(Op));
+ Old2NewVPValues[VPI] = NewR;
+ }
+ }
+
+ VPRecipeBase *LatchTerm = Old2NewVPBBs[OrigLatch]->getTerminator();
+ assert(match(LatchTerm, m_BranchOnCond()) && "Unexpected terminator");
+ VPBuilder LatchBuilder(LatchTerm);
+ VPValue *IVInc = LatchBuilder.createAdd(
+ ScalarIVPHI, One, DebugLoc::getUnknown(), "index.next", {true, false});
+ ScalarIVPHI->addIncoming(IVInc);
+ // The oracle executes scalar iterations up to the parent plan's VF.
+ VPValue *LaneLimit = CreateArg(&Plan.getVF(), "vf");
+ LatchBuilder.createNaryOp(VPInstruction::BranchOnCount, {IVInc, LaneLimit},
+ LatchTerm->getDebugLoc());
+ LatchTerm->eraseFromParent();
+
+ // Return the number of valid leading bytes: (exit IV + 1) * access size.
+ Type *I64Ty = Type::getInt64Ty(OraclePlan->getContext());
+ VPBuilder ExitBuilder(ExitVPBB);
+ VPValue *LaneCount = ExitBuilder.createScalarZExtOrTrunc(
+ ExitBuilder.createAdd(ScalarIVPHI, One, DebugLoc::getUnknown(), "lanes"),
+ I64Ty, DebugLoc::getUnknown());
+ VPValue *ByteCount = ExitBuilder.createOverflowingOp(
+ Instruction::Mul,
+ {LaneCount, OraclePlan->getConstantInt(I64Ty, AccessSize)}, {},
+ DebugLoc::getUnknown(), "bytes");
+ ExitBuilder.createNaryOp(Instruction::Ret, ByteCount);
+
+ VPlanTransforms::convertToConcreteRecipes(*OraclePlan);
+ VPlanTransforms::combineRecipes(*OraclePlan);
+ VPlanTransforms::removeDeadRecipes(*OraclePlan);
+
+ auto OracleArgs = map_to_vector(*NewEntry, [&](VPRecipeBase &R) {
+ return OracleArgToValue.lookup(R.getVPSingleValue());
+ });
+ return {new VPSpeculativeLoadOracleRecipe(std::move(OraclePlan)), OracleArgs};
+}
+
+bool VPlanTransforms::replaceUnsafeLoadsWithSpeculative(
+ VPlan &Plan, Loop *TheLoop, PredicatedScalarEvolution &PSE,
+ DominatorTree &DT, AssumptionCache *AC) {
ScalarEvolution &SE = *PSE.getSE();
const DataLayout &DL = TheLoop->getHeader()->getDataLayout();
- for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(HeaderVPBB)) {
+ auto [HeaderVPBB, LatchVPBB] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
+ auto Bail = [](StringRef Reason) {
+ LLVM_DEBUG(dbgs() << "LV: Not vectorizing: " << Reason << ".\n");
+ return false;
+ };
+
+ auto LoopBody = vp_rpo_plain_cfg_loop_body(HeaderVPBB);
+ SmallVector<VPInstruction *> UnsafeLoads;
+ for (VPBasicBlock *VPBB : LoopBody) {
for (VPRecipeBase &R : *VPBB) {
auto *VPI = dyn_cast<VPInstruction>(&R);
if (!VPI || VPI->getOpcode() != Instruction::Load) {
@@ -1263,9 +1387,85 @@ bool VPlanTransforms::areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB,
TheLoop, SE, DT, AC, &Preds))
continue;
- return false;
+ if (!EnableEarlyExitVectorizationWithSpeculativeLoads)
+ return false;
+ // The oracle may not dereference null, e.g. via embedded null constants.
+ if (NullPointerIsDefined(Load->getFunction(),
+ Load->getPointerAddressSpace()))
+ return Bail("null is a valid address for unsafe load");
+ UnsafeLoads.push_back(VPI);
}
}
+
+ if (UnsafeLoads.empty())
+ return true;
+
+ // Only support primitive element types without padding and with a power-of-2
+ // access size.
+ // TODO: Support pointers and loads with different element types.
+ Type *EltTy = UnsafeLoads.front()->getScalarType();
+ uint64_t AccessSize = DL.getTypeAllocSize(EltTy).getFixedValue();
+ if (EltTy->getPrimitiveSizeInBits() != AccessSize * 8 ||
+ !isPowerOf2_64(AccessSize) ||
+ any_of(UnsafeLoads,
+ [EltTy](VPInstruction *L) { return L->getScalarType() != EltTy; }))
+ return Bail("unsupported element type for speculative loads");
+
+ // TODO: Support non-unit-strided loads.
+ if (any_of(UnsafeLoads, [&](VPInstruction *L) {
+ return vputils::getConstantStride(L->getOperand(0), EltTy, PSE,
+ TheLoop) != 1;
+ }))
+ return Bail("speculative load is not a consecutive access");
+
+ auto EarlyExits =
+ vputils::getEarlyExits(Plan, VPBlockUtils::getPlainCFGMiddleBlock(Plan));
+ if (EarlyExits.size() != 1)
+ return Bail("loop has multiple early exits");
+
+ VPBasicBlock *EarlyExitingVPBB = EarlyExits.front().first;
+ VPDominatorTree VPDT(Plan);
+ for (VPInstruction *VPI : UnsafeLoads)
+ if (!VPDT.dominates(VPI->getParent(), EarlyExitingVPBB) ||
+ !VPDT.dominates(VPI->getParent(), LatchVPBB))
+ return Bail("conditionally executed unsafe load");
+
+ // TODO: Extend oracle logic to support remaining recipes. Freeze and FP
+ // operations may yield different results in the oracle and the vector loop.
+ auto CanReplay = [](const VPRecipeBase &R) {
+ if (auto *IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R))
+ return IV->getScalarType()->isIntegerTy() &&
+ isa<VPIRValue>(IV->getStepValue());
+ auto *VPI = dyn_cast<VPInstruction>(&R);
+ if (!VPI || VPI->hasFastMathFlags())
+ return false;
+ unsigned Opc = VPI->getOpcode();
+ return Instruction::isBinaryOp(Opc) || Instruction::isCast(Opc) ||
+ is_contained<unsigned>(
+ {Instruction::GetElementPtr, Instruction::ICmp,
+ Instruction::Load, Instruction::PHI, Instruction::Select,
+ VPInstruction::BranchOnCond, VPInstruction::Not},
+ Opc);
+ };
+ if (!all_of(LoopBody,
+ [&](VPBasicBlock *VPBB) { return all_of(*VPBB, CanReplay); }))
+ return Bail("loop body cannot be replayed by a speculative-load oracle");
+
+ auto [Oracle, Args] = buildOraclePlan(Plan, AccessSize);
+ cast<VPBasicBlock>(HeaderVPBB->getPredecessors()[0])->appendRecipe(Oracle);
+
+ for (VPInstruction *VPI : UnsafeLoads) {
+ SmallVector<VPValue *> Ops = {VPI->getOperand(0),
+ /*FromEnd=*/Plan.getFalse(), Oracle};
+ append_range(Ops, Args);
+ VPBuilder Builder(VPI);
+ VPValue *SpecLoad = Builder.insert(new VPWidenIntrinsicRecipe(
+ Intrinsic::speculative_load, Ops, VPI->getScalarType(), {}, {},
+ VPI->getDebugLoc()));
+ VPI->replaceAllUsesWith(Builder.createFreeze(SpecLoad, VPI->getDebugLoc()));
+ VPI->eraseFromParent();
+ }
+
return true;
}
@@ -1538,6 +1738,37 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
attachVPCheckBlock(Plan, CondVPV, CheckBlockVPBB, AddBranchWeights);
}
+void VPlanTransforms::attachSpeculativeLoadChecks(
+ VPlan &Plan, ElementCount VF, PredicatedScalarEvolution &PSE, Loop *TheLoop,
+ bool AddBranchWeights) {
+ auto *Oracle = vputils::findSpeculativeLoadOracle(Plan);
+ if (!Oracle)
+ return;
+
+ VPBasicBlock *CheckBlockVPBB = Plan.createVPBasicBlock("spec.load.check");
+ VPBuilder Builder(CheckBlockVPBB);
+ Type *I1Ty = Type::getInt1Ty(Plan.getContext());
+ Type *I64Ty = Type::getInt64Ty(Plan.getContext());
+ const DataLayout &DL = Plan.getDataLayout();
+ VPValue *AllChecksPassed = Plan.getTrue();
+ for (VPUser *U : Oracle->users()) {
+ auto *R = cast<VPWidenIntrinsicRecipe>(U);
+ VPValue *SizeVal = Builder.createElementCount(
+ I64Ty, VF * DL.getTypeStoreSize(R->getScalarType()).getFixedValue());
+ auto *PtrAR = cast<SCEVAddRecExpr>(
+ vputils::getSCEVExprForVPValue(R->getOperand(0), PSE, TheLoop));
+ VPValue *StartPtr =
+ vputils::getOrCreateVPValueForSCEVExpr(Plan, PtrAR->getStart());
+ VPValue *IsSafe = Builder.createScalarIntrinsic(
+ Intrinsic::can_load_speculatively, {StartPtr, SizeVal}, I1Ty,
+ DebugLoc::getUnknown());
+ AllChecksPassed = Builder.createAnd(AllChecksPassed, IsSafe);
+ }
+
+ attachVPCheckBlock(Plan, Builder.createNot(AllChecksPassed), CheckBlockVPBB,
+ AddBranchWeights);
+}
+
void VPlanTransforms::addMinimumIterationCheck(
VPlan &Plan, ElementCount VF, unsigned UF,
ElementCount MinProfitableTripCount, bool RequiresScalarEpilogue,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 4b68e3e71afea9..900145e9bad5dc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -13,10 +13,12 @@
#include "LoopVectorizationPlanner.h"
#include "VPlan.h"
+#include "VPlanCFG.h"
#include "VPlanHelpers.h"
#include "VPlanPatternMatch.h"
#include "VPlanUtils.h"
#include "llvm/ADT/APFloat.h"
+#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/SmallVectorExtras.h"
@@ -96,6 +98,7 @@ bool VPRecipeBase::mayWriteToMemory() const {
case VPScalarIVStepsSC:
case VPPredInstPHISC:
case VPExpandSCEVSC:
+ case VPSpeculativeLoadOracleSC:
return false;
case VPBlendSC:
case VPReductionEVLSC:
@@ -151,6 +154,7 @@ bool VPRecipeBase::mayReadFromMemory() const {
case VPWidenStoreEVLSC:
case VPWidenStoreSC:
case VPExpandSCEVSC:
+ case VPSpeculativeLoadOracleSC:
return false;
case VPBlendSC:
case VPReductionEVLSC:
@@ -188,10 +192,11 @@ bool VPRecipeBase::mayHaveSideEffects() const {
case VPPredInstPHISC:
case VPVectorEndPointerSC:
case VPExpandSCEVSC:
+ case VPSpeculativeLoadOracleSC:
return false;
case VPInstructionSC: {
auto *VPI = cast<VPInstruction>(this);
- return mayWriteToMemory() ||
+ return mayWriteToMemory() || VPI->getOpcode() == Instruction::Ret ||
VPI->getOpcode() == VPInstruction::BranchOnCount ||
VPI->getOpcode() == VPInstruction::BranchOnCond ||
VPI->getOpcode() == VPInstruction::BranchOnTwoConds;
@@ -499,6 +504,7 @@ Type *llvm::computeScalarTypeForInstruction(unsigned Opcode,
for (unsigned Idx = 1; Idx != Operands.size(); ++Idx)
AssertOperandType(Idx, Op0Ty);
return Type::getVoidTy(Ctx);
+ case Instruction::Ret:
case Instruction::Store:
return Type::getVoidTy(Ctx);
case Instruction::ICmp:
@@ -642,8 +648,10 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
switch (Opcode) {
case VPInstruction::StepVector:
case VPInstruction::IncomingAliasMask:
+ case VPInstruction::LiveIn:
return 0;
case Instruction::Alloca:
+ case Instruction::Ret:
case Instruction::ExtractValue:
case Instruction::Freeze:
case Instruction::Load:
@@ -882,6 +890,12 @@ Value *VPInstruction::generate(VPTransformState &State) {
{AVL, VFArg, Builder.getTrue()});
return EVL;
}
+ case Instruction::Ret: {
+ auto *Ret = Builder.CreateRet(State.get(getOperand(0), /*IsScalar=*/true));
+ // Remove the temporary unreachable terminator.
+ Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
+ return Ret;
+ }
case VPInstruction::BranchOnCond: {
Value *Cond = State.get(getOperand(0), VPLane(0));
// Replace the temporary unreachable terminator with a new conditional
@@ -1592,6 +1606,7 @@ bool VPInstruction::isSingleScalar() const {
case VPInstruction::ExplicitVectorLength:
case VPInstruction::ResumeForEpilogue:
case VPInstruction::Intrinsic:
+ case VPInstruction::LiveIn:
return true;
default:
return Instruction::isCast(getOpcode());
@@ -1676,6 +1691,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
Instruction::isUnaryOp(getOpcode()) || Instruction::isCast(getOpcode()))
return false;
switch (getOpcode()) {
+ case Instruction::Ret:
case Instruction::ExtractValue:
case Instruction::InsertValue:
case Instruction::GetElementPtr:
@@ -1702,6 +1718,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
case VPInstruction::ActiveLaneMask:
case VPInstruction::WideActiveLaneMask:
case VPInstruction::IncomingAliasMask:
+ case VPInstruction::LiveIn:
case VPInstruction::ExitingIVValue:
case VPInstruction::ExplicitVectorLength:
case VPInstruction::FirstActiveLane:
@@ -1749,6 +1766,7 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const {
case VPInstruction::ExtractLastActive:
return Op == getOperand(0);
case Instruction::PHI:
+ case Instruction::Ret:
return true;
case Instruction::FCmp:
case Instruction::ICmp:
@@ -1835,6 +1853,9 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::WideActiveLaneMask:
O << "wide active lane mask";
break;
+ case VPInstruction::LiveIn:
+ O << "live-in";
+ break;
case VPInstruction::IncomingAliasMask:
O << "incoming-alias-mask";
break;
@@ -3619,6 +3640,72 @@ InstructionCost VPReductionRecipe::computeCost(ElementCount VF,
Ctx.CostKind);
}
+VPSpeculativeLoadOracleRecipe::VPSpeculativeLoadOracleRecipe(
+ std::unique_ptr<VPlan> Oracle)
+ : VPSingleDefRecipe(VPRecipeBase::VPSpeculativeLoadOracleSC, {},
+ PointerType::getUnqual(Oracle->getContext())),
+ OraclePlan(std::move(Oracle)) {}
+
+VPSpeculativeLoadOracleRecipe *VPSpeculativeLoadOracleRecipe::clone() {
+ return new VPSpeculativeLoadOracleRecipe(
+ std::unique_ptr<VPlan>(OraclePlan->duplicate()));
+}
+
+void VPSpeculativeLoadOracleRecipe::execute(VPTransformState &State) {
+ VPlan &Plan = *OraclePlan;
+ LLVMContext &Ctx = Plan.getContext();
+ auto ParamTypes = map_to_vector(*Plan.getEntry(), [](VPRecipeBase &R) {
+ return R.getVPSingleValue()->getScalarType();
+ });
+
+ Module *M = State.Builder.GetInsertBlock()->getModule();
+ // TODO: Unsupported under a CGSCC adaptor, as LazyCallGraph is not updated.
+ Function *OracleFn = Function::Create(
+ FunctionType::get(Type::getInt64Ty(Ctx), ParamTypes, /*isVarArg=*/false),
+ GlobalValue::InternalLinkage, "speculativeLoadOracle", M);
+ OracleFn->setMemoryEffects(MemoryEffects::argMemOnly(ModRefInfo::Ref));
+ for (Attribute::AttrKind Kind :
+ {Attribute::NoInline, Attribute::OptimizeNone, Attribute::NoUnwind,
+ Attribute::NoSync, Attribute::MustProgress, Attribute::WillReturn})
+ OracleFn->addFnAttr(Kind);
+
+ BasicBlock *EntryBB = BasicBlock::Create(Ctx, "oracle.entry", OracleFn);
+ IRBuilder<> OracleBuilder(EntryBB);
+ OracleBuilder.SetInsertPoint(OracleBuilder.CreateUnreachable());
+
+ // Execute the plan with VF=1.
+ LoopInfo OracleLI;
+ OracleLI.analyze(OracleFn);
+ VPTransformState OracleState(State.TTI, ElementCount::getFixed(1), &OracleLI,
+ /*DT=*/nullptr, /*AC=*/nullptr, OracleBuilder,
+ &Plan, /*CurrentParentLoop=*/nullptr);
+ OracleState.CFG.PrevBB = EntryBB;
+ OracleState.CFG.VPBB2IRBB[Plan.getEntry()] = EntryBB;
+
+ for (auto [R, Arg] : zip_equal(*Plan.getEntry(), OracleFn->args())) {
+ Arg.setName(cast<VPInstruction>(R).getName());
+ OracleState.set(R.getVPSingleValue(), &Arg, /*IsScalar=*/true);
+ }
+ ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
+ Plan.getEntry());
+ for (VPBlockBase *Block : drop_begin(RPOT))
+ Block->execute(&OracleState);
+ OracleState.fixupHeaderPhis();
+
+ State.set(this, OracleFn, /*IsScalar=*/true);
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void VPSpeculativeLoadOracleRecipe::printRecipe(raw_ostream &O,
+ const Twine &Indent,
+ VPSlotTracker &Tracker) const {
+ O << Indent << "SPECULATIVE-LOAD-ORACLE ";
+ printAsOperand(O, Tracker);
+ O << '\n';
+ OraclePlan->print(O, Indent + " ");
+}
+#endif
+
VPExpressionRecipe::VPExpressionRecipe(
ExpressionTypes ExpressionType,
ArrayRef<VPSingleDefRecipe *> ExpressionRecipes)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 13abd173502e9e..3a98a8b1a7d135 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2281,6 +2281,10 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
C->second == Instruction::ExtractValue)))
return false;
+ // Each live-in represents a distinct external input.
+ if (match(Def, m_VPInstruction<VPInstruction::LiveIn>()))
+ return false;
+
// Widened loads (including the EVL variant) are handled, as cse() only
// reuses them within a block with no intervening memory write. Any other
// memory access is rejected.
@@ -3367,11 +3371,10 @@ bool VPlanTransforms::handleUncountableEarlyExits(
auto *MiddleVPBB = VPBlockUtils::getPlainCFGMiddleBlock(Plan);
auto [HeaderVPBB, LatchVPBB] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
- // Dereferenceability is checked separately for uncountable exit loops with
- // stores, as only the loads contributing to the exit condition need to
- // be checked.
+ // Dereferenceability is checked separately for uncountable exit loops without
+ // stores; loads that may fault are replaced by speculative loads, if allowed.
if (Style == UncountableExitStyle::ReadOnly &&
- !areAllLoadsDereferenceable(HeaderVPBB, TheLoop, PSE, DT, AC)) {
+ !replaceUnsafeLoadsWithSpeculative(Plan, TheLoop, PSE, DT, AC)) {
reportVectorizationFailure(
"Auto-vectorization of early exit loops with potentially "
"faulting loads is not supported",
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index fd62693d6068be..c8f627bb919ef4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -243,6 +243,12 @@ struct VPlanTransforms {
static void modelGeneratedMainLoopBlocks(VPlan &EpiPlan, VPlan &MainPlan,
VPIRBasicBlock *EnteredFrom);
+ /// Attach @llvm.can.load.speculatively checks for \p Plan's speculative
+ /// loads, bypassing the vector loop if any fails.
+ static void attachSpeculativeLoadChecks(VPlan &Plan, ElementCount VF,
+ PredicatedScalarEvolution &PSE,
+ Loop *TheLoop, bool AddBranchWeights);
+
/// Replaces the VPInstructions in \p Plan with corresponding
/// widen recipes. Returns false if any VPInstructions could not be converted
/// to a wide recipe if needed. Uses \p PSE to detect contiguous memory
@@ -380,14 +386,14 @@ struct VPlanTransforms {
/// Remove dead recipes from \p Plan.
static void removeDeadRecipes(VPlan &Plan);
- /// Check if all loads in the loop are dereferenceable. Iterates over the
- /// loop body blocks reachable from \p HeaderVPBB. Returns false if any
- /// non-dereferenceable load is found.
- static bool areAllLoadsDereferenceable(VPBasicBlock *HeaderVPBB,
- Loop *TheLoop,
- PredicatedScalarEvolution &PSE,
- DominatorTree &DT,
- AssumptionCache *AC);
+ /// Replace loads that may fault with @llvm.speculative.load, backed by an
+ /// oracle plan replaying \p Plan's exit conditions. Must run before the early
+ /// exits are flattened. Returns false if a load cannot be replaced, or if any
+ /// load may fault and speculative loads are not enabled.
+ static bool replaceUnsafeLoadsWithSpeculative(VPlan &Plan, Loop *TheLoop,
+ PredicatedScalarEvolution &PSE,
+ DominatorTree &DT,
+ AssumptionCache *AC);
/// Update \p Plan to account for uncountable early exits by introducing
/// appropriate branching logic in the latch that handles early exits and the
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index fbc657ad16854d..73d98c4a7a0e3d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -478,8 +478,8 @@ bool vputils::isSingleScalar(const VPValue *VPV) {
all_of(VPI->operands(), isSingleScalar));
if (auto *RR = dyn_cast<VPReductionRecipe>(VPV))
return !RR->isPartialReduction();
- if (isa<VPVectorPointerRecipe, VPVectorEndPointerRecipe, VPDerivedIVRecipe>(
- VPV))
+ if (isa<VPVectorPointerRecipe, VPVectorEndPointerRecipe, VPDerivedIVRecipe,
+ VPSpeculativeLoadOracleRecipe>(VPV))
return true;
if (auto *Expr = dyn_cast<VPExpressionRecipe>(VPV))
return Expr->isVectorToScalar();
@@ -616,6 +616,13 @@ VPValue *vputils::findIncomingAliasMask(const VPlan &Plan) {
return nullptr;
}
+VPSpeculativeLoadOracleRecipe *vputils::findSpeculativeLoadOracle(VPlan &Plan) {
+ for (VPRecipeBase &R : *Plan.getVectorPreheader())
+ if (auto *Oracle = dyn_cast<VPSpeculativeLoadOracleRecipe>(&R))
+ return Oracle;
+ return nullptr;
+}
+
SmallVector<std::pair<VPBasicBlock *, VPIRBasicBlock *>>
vputils::getEarlyExits(const VPlan &Plan, const VPBlockBase *MiddleVPBB) {
SmallVector<std::pair<VPBasicBlock *, VPIRBasicBlock *>> Exits;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 967e612ab71147..26357e64af9ffe 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -180,6 +180,10 @@ VPInstruction *findComputeReductionResult(VPReductionPHIRecipe *PhiR);
/// Finds the incoming alias-mask within the vector preheader.
VPValue *findIncomingAliasMask(const VPlan &Plan);
+/// Finds the speculative-load oracle in \p Plan's vector preheader, if any.
+/// Requires loop regions to be created.
+VPSpeculativeLoadOracleRecipe *findSpeculativeLoadOracle(VPlan &Plan);
+
/// Returns the (early exiting block, exit block) pairs of \p Plan, i.e. all
/// edges to an exit block that do not come from \p MiddleVPBB.
SmallVector<std::pair<VPBasicBlock *, VPIRBasicBlock *>>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 6fbd53d5293807..6873edd65754bf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -320,12 +320,32 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
return false;
}
}
+ if (const auto *Oracle = dyn_cast<VPSpeculativeLoadOracleRecipe>(&R)) {
+ if (!verifyVPlanIsValid(Oracle->getOraclePlan())) {
+ errs() << "Invalid speculative-load oracle plan!\n";
+ return false;
+ }
+ }
if (const auto *VPI = dyn_cast<VPInstruction>(&R)) {
switch (VPI->getOpcode()) {
+ case Instruction::Ret:
+ if (&R != &VPBB->back() || VPBB->getParent() ||
+ VPBB->getNumSuccessors() != 0) {
+ errs() << "Return must terminate a top-level block without "
+ "successors!\n";
+ return false;
+ }
+ break;
case VPInstruction::LastActiveLane:
if (!verifyLastActiveLaneRecipe(*VPI))
return false;
break;
+ case VPInstruction::LiveIn:
+ if (VPBB != VPBB->getPlan()->getEntry()) {
+ errs() << "Live-in must be in the plan's entry block!\n";
+ return false;
+ }
+ break;
default:
break;
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-basic.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-basic.ll
index 8ca8b9ea7737cf..8423bbf41e6130 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-basic.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-basic.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --include-generated-funcs --version 6
-; RUN: opt -p loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
+; RUN: opt -p loop-vectorize -enable-early-exit-vectorization-with-speculative-loads -force-vector-width=4 -S %s | FileCheck %s
target triple = "arm64-apple-macosx"
@@ -151,24 +151,90 @@ exit:
ret i32 0
}
+; The loads use a derived induction starting at 10, the runtime check start
+; pointers are expanded from SCEV.
+define i64 @derived_induction_start(ptr %A, ptr %B, i64 %n) {
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %iv.off = phi i64 [ 10, %entry ], [ %iv.off.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv.off
+ %ld.A = load i8, ptr %gep.A, align 1
+ %gep.B = getelementptr inbounds i8, ptr %B, i64 %iv.off
+ %ld.B = load i8, ptr %gep.B, align 1
+ %cmp = icmp ne i8 %ld.A, %ld.B
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %iv.off.next = add nuw nsw i64 %iv.off, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
; CHECK-LABEL: define i64 @find_first_mismatch(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[SPEC_LOAD_CHECK:.*]]
+; CHECK: [[SPEC_LOAD_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[A]], i64 4)
+; CHECK-NEXT: [[TMP1:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[B]], i64 4)
+; CHECK-NEXT: [[TMP2:%.*]] = and i1 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = xor i1 [[TMP2]], true
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP4:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP4]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
-; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[TMP6:%.*]] = call <4 x i8> (ptr, i1, ...) @llvm.speculative.load.v4i8.p0(ptr [[GEP_A]], i1 false, ptr @speculativeLoadOracle, ptr [[A]], ptr [[B]], i64 [[IV]], i64 4)
+; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i8> [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[TMP9:%.*]] = call <4 x i8> (ptr, i1, ...) @llvm.speculative.load.v4i8.p0(ptr [[TMP8]], i1 false, ptr @speculativeLoadOracle, ptr [[A]], ptr [[B]], i64 [[IV]], i64 4)
+; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i8> [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp ne <4 x i8> [[TMP7]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP12]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP12]], i1 false)
+; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[IV]], [[TMP15]]
+; CHECK-NEXT: br label %[[EARLY_EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[SPEC_LOAD_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV1]]
+; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[GEP_A1]], align 1
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV1]]
; CHECK-NEXT: [[LD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[LD_A]], [[LD_B]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT]], label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EARLY_EXIT]]:
-; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV1]], %[[LOOP_HEADER]] ], [ [[TMP16]], %[[VECTOR_EARLY_EXIT]] ]
; CHECK-NEXT: ret i64 [[IV_LCSSA]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i64 -1
@@ -177,19 +243,54 @@ exit:
; CHECK-LABEL: define i64 @find_first_mismatch_no_live_out(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[SPEC_LOAD_CHECK:.*]]
+; CHECK: [[SPEC_LOAD_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[A]], i64 4)
+; CHECK-NEXT: [[TMP1:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[B]], i64 4)
+; CHECK-NEXT: [[TMP2:%.*]] = and i1 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = xor i1 [[TMP2]], true
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP4:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP4]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
-; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[TMP6:%.*]] = call <4 x i8> (ptr, i1, ...) @llvm.speculative.load.v4i8.p0(ptr [[GEP_A]], i1 false, ptr @speculativeLoadOracle.1, ptr [[A]], ptr [[B]], i64 [[IV]], i64 4)
+; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i8> [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[TMP9:%.*]] = call <4 x i8> (ptr, i1, ...) @llvm.speculative.load.v4i8.p0(ptr [[TMP8]], i1 false, ptr @speculativeLoadOracle.1, ptr [[A]], ptr [[B]], i64 [[IV]], i64 4)
+; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i8> [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp ne <4 x i8> [[TMP7]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP12]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: br label %[[EARLY_EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[SPEC_LOAD_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV1]]
+; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[GEP_A1]], align 1
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV1]]
; CHECK-NEXT: [[LD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[LD_A]], [[LD_B]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT]], label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EARLY_EXIT]]:
; CHECK-NEXT: ret i64 1
; CHECK: [[EXIT]]:
@@ -199,38 +300,112 @@ exit:
; CHECK-LABEL: define i64 @early_and_latch_exit_to_same_block(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[SPEC_LOAD_CHECK:.*]]
+; CHECK: [[SPEC_LOAD_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[A]], i64 4)
+; CHECK-NEXT: [[TMP1:%.*]] = xor i1 [[TMP0]], true
+; CHECK-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP2]]
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = call <4 x i8> (ptr, i1, ...) @llvm.speculative.load.v4i8.p0(ptr [[GEP_A]], i1 false, ptr @speculativeLoadOracle.2, ptr [[A]], i64 [[IV]], i64 4)
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i8> [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i8> [[TMP5]], splat (i8 42)
+; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP7]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP7]], i1 false)
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[IV]], [[TMP10]]
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[SPEC_LOAD_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP_HEADER1:.*]]
+; CHECK: [[LOOP_HEADER1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV1]]
+; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[GEP_A1]], align 1
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[LD_A]], 42
-; CHECK-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[EXIT]], label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER1]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ], [ -1, %[[LOOP_LATCH]] ]
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[IV1]], %[[LOOP_HEADER1]] ], [ -1, %[[LOOP_LATCH]] ], [ -1, %[[MIDDLE_BLOCK]] ], [ [[TMP11]], %[[VECTOR_EARLY_EXIT]] ]
; CHECK-NEXT: ret i64 [[RES]]
;
;
; CHECK-LABEL: define ptr @address_live_out(
; CHECK-SAME: ptr [[P:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[SPEC_LOAD_CHECK:.*]]
+; CHECK: [[SPEC_LOAD_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[P]], i64 4)
+; CHECK-NEXT: [[TMP1:%.*]] = xor i1 [[TMP0]], true
+; CHECK-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP2]]
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[IV]], 2
+; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[IV]], 3
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[IV]]
-; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP4]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x ptr> poison, ptr [[GEP]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x ptr> [[TMP10]], ptr [[TMP7]], i64 1
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x ptr> [[TMP11]], ptr [[TMP8]], i64 2
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x ptr> [[TMP12]], ptr [[TMP9]], i64 3
+; CHECK-NEXT: [[TMP14:%.*]] = call <4 x i8> (ptr, i1, ...) @llvm.speculative.load.v4i8.p0(ptr [[GEP]], i1 false, ptr @speculativeLoadOracle.3, ptr [[P]], i64 [[IV]], i64 4)
+; CHECK-NEXT: [[TMP15:%.*]] = freeze <4 x i8> [[TMP14]]
+; CHECK-NEXT: [[TMP16:%.*]] = icmp eq <4 x i8> [[TMP15]], zeroinitializer
+; CHECK-NEXT: [[TMP17:%.*]] = freeze <4 x i1> [[TMP16]]
+; CHECK-NEXT: [[TMP18:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP17]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP18]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP17]], i1 false)
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x ptr> [[TMP13]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EARLY_EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[SPEC_LOAD_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP_HEADER1:.*]]
+; CHECK: [[LOOP_HEADER1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[IV1]]
+; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[GEP1]], align 1
; CHECK-NEXT: [[C:%.*]] = icmp eq i8 [[L]], 0
-; CHECK-NEXT: br i1 [[C]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK-NEXT: br i1 [[C]], label %[[EARLY_EXIT]], label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER1]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[EARLY_EXIT]]:
-; CHECK-NEXT: [[GEP_LCSSA:%.*]] = phi ptr [ [[GEP]], %[[LOOP_HEADER]] ]
+; CHECK-NEXT: [[GEP_LCSSA:%.*]] = phi ptr [ [[GEP1]], %[[LOOP_HEADER1]] ], [ [[TMP20]], %[[VECTOR_EARLY_EXIT]] ]
; CHECK-NEXT: ret ptr [[GEP_LCSSA]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret ptr null
@@ -239,20 +414,63 @@ exit:
; CHECK-LABEL: define i64 @index_cast_live_out(
; CHECK-SAME: ptr [[P:%.*]], i32 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[SPEC_LOAD_CHECK:.*]]
+; CHECK: [[SPEC_LOAD_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[P]], i64 4)
+; CHECK-NEXT: [[TMP1:%.*]] = xor i1 [[TMP0]], true
+; CHECK-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = and i32 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[TMP2]]
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[IV]], 2
+; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[IV]], 3
; CHECK-NEXT: [[IDX:%.*]] = zext i32 [[IV]] to i64
+; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP3]] to i64
+; CHECK-NEXT: [[TMP8:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP5]] to i64
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i64> poison, i64 [[IDX]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i64> [[TMP10]], i64 [[TMP7]], i64 1
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i64> [[TMP11]], i64 [[TMP8]], i64 2
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> [[TMP12]], i64 [[TMP9]], i64 3
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[IDX]]
-; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-NEXT: [[TMP15:%.*]] = call <4 x i8> (ptr, i1, ...) @llvm.speculative.load.v4i8.p0(ptr [[GEP]], i1 false, ptr @speculativeLoadOracle.4, ptr [[P]], i32 [[IV]], i32 4)
+; CHECK-NEXT: [[TMP16:%.*]] = freeze <4 x i8> [[TMP15]]
+; CHECK-NEXT: [[TMP17:%.*]] = icmp eq <4 x i8> [[TMP16]], zeroinitializer
+; CHECK-NEXT: [[TMP18:%.*]] = freeze <4 x i1> [[TMP17]]
+; CHECK-NEXT: [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP18]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[IV]], 4
+; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP19]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP18]], i1 false)
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i64> [[TMP13]], i64 [[FIRST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EARLY_EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[SPEC_LOAD_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP_HEADER1:.*]]
+; CHECK: [[LOOP_HEADER1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[IDX1:%.*]] = zext i32 [[IV1]] to i64
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i8, ptr [[P]], i64 [[IDX1]]
+; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[GEP1]], align 1
; CHECK-NEXT: [[C:%.*]] = icmp eq i8 [[L]], 0
-; CHECK-NEXT: br i1 [[C]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK-NEXT: br i1 [[C]], label %[[EARLY_EXIT]], label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER1]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[EARLY_EXIT]]:
-; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i64 [ [[IDX]], %[[LOOP_HEADER]] ]
+; CHECK-NEXT: [[IDX_LCSSA:%.*]] = phi i64 [ [[IDX1]], %[[LOOP_HEADER1]] ], [ [[TMP21]], %[[VECTOR_EARLY_EXIT]] ]
; CHECK-NEXT: ret i64 [[IDX_LCSSA]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i64 -1
@@ -260,25 +478,241 @@ exit:
;
; CHECK-LABEL: define i32 @masked_index_broadcast(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
-; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT: [[ADD:%.*]] = add i64 [[IV]], [[N]]
-; CHECK-NEXT: [[CMP:%.*]] = icmp ugt i64 [[ADD]], 0
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER1:.*]]
+; CHECK: [[LOOP_HEADER1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[ADD1:%.*]] = add i64 [[IV1]], [[N]]
+; CHECK-NEXT: [[CMP:%.*]] = icmp ugt i64 [[ADD1]], 0
; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP_LATCH]], label %[[BODY:.*]]
; CHECK: [[BODY]]:
-; CHECK-NEXT: [[IDX:%.*]] = and i64 [[ADD]], 4294967295
+; CHECK-NEXT: [[IDX:%.*]] = and i64 [[ADD1]], 4294967295
; CHECK-NEXT: [[GEP:%.*]] = getelementptr [8 x i8], ptr [[A]], i64 [[IDX]]
; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 8
; CHECK-NEXT: [[C:%.*]] = icmp eq i64 [[L]], 0
; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH]], label %[[EARLY_EXIT:.*]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV1]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV1]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER1]]
; CHECK: [[EARLY_EXIT]]:
; CHECK-NEXT: ret i32 1
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i32 0
;
+;
+; CHECK-LABEL: define i64 @derived_induction_start(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr nuw i8, ptr [[B]], i64 10
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr nuw i8, ptr [[A]], i64 10
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[SPEC_LOAD_CHECK:.*]]
+; CHECK: [[SPEC_LOAD_CHECK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[TMP1]], i64 4)
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[TMP0]], i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = and i1 [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = xor i1 [[TMP4]], true
+; CHECK-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP6:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = add i64 10, [[N_VEC]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = add nuw i64 10, [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = call <4 x i8> (ptr, i1, ...) @llvm.speculative.load.v4i8.p0(ptr [[TMP9]], i1 false, ptr @speculativeLoadOracle.5, ptr [[A]], ptr [[B]], i64 [[TMP8]], i64 4)
+; CHECK-NEXT: [[TMP11:%.*]] = freeze <4 x i8> [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP8]]
+; CHECK-NEXT: [[TMP13:%.*]] = call <4 x i8> (ptr, i1, ...) @llvm.speculative.load.v4i8.p0(ptr [[TMP12]], i1 false, ptr @speculativeLoadOracle.5, ptr [[A]], ptr [[B]], i64 [[TMP8]], i64 4)
+; CHECK-NEXT: [[TMP14:%.*]] = freeze <4 x i8> [[TMP13]]
+; CHECK-NEXT: [[TMP15:%.*]] = icmp ne <4 x i8> [[TMP11]], [[TMP14]]
+; CHECK-NEXT: [[TMP16:%.*]] = freeze <4 x i1> [[TMP15]]
+; CHECK-NEXT: [[TMP17:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP17]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP19:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP16]], i1 false)
+; CHECK-NEXT: [[TMP20:%.*]] = add i64 [[INDEX]], [[TMP19]]
+; CHECK-NEXT: br label %[[EARLY_EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[SPEC_LOAD_CHECK]] ]
+; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 10, %[[ENTRY]] ], [ 10, %[[SPEC_LOAD_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[IV_OFF:%.*]] = phi i64 [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[IV_OFF_NEXT:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV_OFF]]
+; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV_OFF]]
+; CHECK-NEXT: [[LD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[LD_A]], [[LD_B]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_OFF_NEXT]] = add nuw nsw i64 [[IV_OFF]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ], [ [[TMP20]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: ret i64 [[IV_LCSSA]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 -1
+;
+;
+; CHECK-LABEL: define internal i64 @speculativeLoadOracle(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[IV_START:%.*]], i64 [[VF:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: [[ORACLE_ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ORACLE_ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[IV_START]], [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[TMP10]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ne i8 [[TMP5]], [[TMP8]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[ORACLE_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[VF]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[ORACLE_EXIT]], label %[[LOOP_HEADER]]
+; CHECK: [[ORACLE_EXIT]]:
+; CHECK-NEXT: [[LANES:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: ret i64 [[LANES]]
+;
+;
+; CHECK-LABEL: define internal i64 @speculativeLoadOracle.1(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[IV_START:%.*]], i64 [[VF:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ORACLE_ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ORACLE_ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[IV_START]], [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[TMP10]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ne i8 [[TMP5]], [[TMP8]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[ORACLE_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[VF]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[ORACLE_EXIT]], label %[[LOOP_HEADER]]
+; CHECK: [[ORACLE_EXIT]]:
+; CHECK-NEXT: [[LANES:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: ret i64 [[LANES]]
+;
+;
+; CHECK-LABEL: define internal i64 @speculativeLoadOracle.2(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[IV_START:%.*]], i64 [[VF:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ORACLE_ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ORACLE_ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[IV_START]], [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i8 [[TMP3]], 42
+; CHECK-NEXT: br i1 [[TMP4]], label %[[ORACLE_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[VF]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[ORACLE_EXIT]], label %[[LOOP_HEADER]]
+; CHECK: [[ORACLE_EXIT]]:
+; CHECK-NEXT: [[LANES:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: ret i64 [[LANES]]
+;
+;
+; CHECK-LABEL: define internal i64 @speculativeLoadOracle.3(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[IV_START:%.*]], i64 [[VF:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ORACLE_ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ORACLE_ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[IV_START]], [[INDEX]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP3:%.*]] = load i8, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i8 [[TMP3]], 0
+; CHECK-NEXT: br i1 [[TMP4]], label %[[ORACLE_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[VF]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[ORACLE_EXIT]], label %[[LOOP_HEADER]]
+; CHECK: [[ORACLE_EXIT]]:
+; CHECK-NEXT: [[LANES:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: ret i64 [[LANES]]
+;
+;
+; CHECK-LABEL: define internal i64 @speculativeLoadOracle.4(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[IV_START:%.*]], i32 [[VF:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ORACLE_ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[ORACLE_ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[IV_START]], [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP9]]
+; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i8 [[TMP4]], 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[ORACLE_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[VF]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[ORACLE_EXIT]], label %[[LOOP_HEADER]]
+; CHECK: [[ORACLE_EXIT]]:
+; CHECK-NEXT: [[LANES:%.*]] = add i32 [[INDEX]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[LANES]] to i64
+; CHECK-NEXT: ret i64 [[TMP7]]
+;
+;
+; CHECK-LABEL: define internal i64 @speculativeLoadOracle.5(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[IV_START:%.*]], i64 [[VF:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ORACLE_ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ORACLE_ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[IV_START]], [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[TMP9]], align 1
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i8, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = icmp ne i8 [[TMP10]], [[TMP8]]
+; CHECK-NEXT: br i1 [[TMP5]], label %[[ORACLE_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[VF]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[ORACLE_EXIT]], label %[[LOOP_HEADER]]
+; CHECK: [[ORACLE_EXIT]]:
+; CHECK-NEXT: [[LANES:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: ret i64 [[LANES]]
+;
+;.
+; CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+; CHECK: attributes #[[ATTR1]] = { mustprogress noinline nosync nounwind optnone willreturn memory(argmem: read) }
+; CHECK: attributes #[[ATTR2:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: read) }
+; CHECK: attributes #[[ATTR3:[0-9]+]] = { nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) }
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-forced-vf.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-forced-vf.ll
new file mode 100644
index 00000000000000..16a527fd6259ae
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-forced-vf.ll
@@ -0,0 +1,88 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -p loop-vectorize -enable-early-exit-vectorization-with-speculative-loads -force-vector-width=8 \
+; RUN: -epilogue-vectorization-force-VF=4 -S %s | FileCheck %s
+
+target triple = "arm64-apple-macosx"
+
+; The forced VF 8 is ignored, because the speculative <8 x i32> load exceeds 16
+; bytes and has invalid cost. A VF must still be picked with a forced epilogue VF.
+define i64 @forced_vf_invalid_cost(ptr %A, i64 %n) {
+; CHECK-LABEL: define i64 @forced_vf_invalid_cost(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[SPEC_LOAD_CHECK:.*]]
+; CHECK: [[SPEC_LOAD_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[A]], i64 16)
+; CHECK-NEXT: [[TMP1:%.*]] = xor i1 [[TMP0]], true
+; CHECK-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP2]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr [[TMP3]], i1 false, ptr @speculativeLoadOracle, ptr [[A]], i64 [[INDEX]], i64 4)
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i32> [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i32> [[TMP5]], splat (i32 42)
+; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP7]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP7]], i1 false)
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]]
+; CHECK-NEXT: br label %[[EARLY_EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[SPEC_LOAD_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LD_A:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[LD_A]], 42
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ], [ [[TMP11]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: ret i64 [[IV_LCSSA]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 -1
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv
+ %ld.A = load i32, ptr %gep.A, align 4
+ %cmp = icmp eq i32 %ld.A, 42
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-irregular-type.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-irregular-type.ll
new file mode 100644
index 00000000000000..e1d5a5c6d1cd8a
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-irregular-type.ll
@@ -0,0 +1,97 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -p loop-vectorize -enable-early-exit-vectorization-with-speculative-loads -force-vector-width=4 -S %s | FileCheck %s
+
+; The alloc sizes of i16 and double are larger than their store sizes, so
+; consecutive elements are not contiguous in memory. Make sure loops with such
+; potentially faulting loads are not vectorized using speculative loads.
+
+target datalayout = "e-i16:32-f64:128-n32:64"
+target triple = "arm64-apple-macosx"
+
+define i64 @load_i16_irregular(ptr %A, i64 %n) {
+; CHECK-LABEL: define i64 @load_i16_irregular(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LD_A:%.*]] = load i16, ptr [[GEP_A]], align 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i16 [[LD_A]], 1
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP]] ]
+; CHECK-NEXT: ret i64 [[IV_LCSSA]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 -1
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i16, ptr %A, i64 %iv
+ %ld.A = load i16, ptr %gep.A, align 1
+ %cmp = icmp eq i16 %ld.A, 1
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+define i64 @load_double_irregular(ptr %A, i64 %n) {
+; CHECK-LABEL: define i64 @load_double_irregular(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LD_A:%.*]] = load double, ptr [[GEP_A]], align 8
+; CHECK-NEXT: [[BITS:%.*]] = bitcast double [[LD_A]] to i64
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[BITS]], 0
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP]] ]
+; CHECK-NEXT: ret i64 [[IV_LCSSA]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 -1
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds double, ptr %A, i64 %iv
+ %ld.A = load double, ptr %gep.A, align 8
+ %bits = bitcast double %ld.A to i64
+ %cmp = icmp eq i64 %bits, 0
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-load-types.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-load-types.ll
index 757856f2fbeafe..d580ac028e7841 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-load-types.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-load-types.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --include-generated-funcs --version 6
-; RUN: opt -p loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
+; RUN: opt -p loop-vectorize -enable-early-exit-vectorization-with-speculative-loads -force-vector-width=4 -S %s | FileCheck %s
target triple = "arm64-apple-macosx"
@@ -124,21 +124,58 @@ exit:
; CHECK-LABEL: define i64 @load_i16(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[SPEC_LOAD_CHECK:.*]]
+; CHECK: [[SPEC_LOAD_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[A]], i64 8)
+; CHECK-NEXT: [[TMP1:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[B]], i64 8)
+; CHECK-NEXT: [[TMP2:%.*]] = and i1 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = xor i1 [[TMP2]], true
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP4:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP4]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT: [[LD_A:%.*]] = load i16, ptr [[GEP_A]], align 2
-; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[TMP6:%.*]] = call <4 x i16> (ptr, i1, ...) @llvm.speculative.load.v4i16.p0(ptr [[GEP_A]], i1 false, ptr @speculativeLoadOracle, ptr [[A]], ptr [[B]], i64 [[IV]], i64 4)
+; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i16> [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[TMP9:%.*]] = call <4 x i16> (ptr, i1, ...) @llvm.speculative.load.v4i16.p0(ptr [[TMP8]], i1 false, ptr @speculativeLoadOracle, ptr [[A]], ptr [[B]], i64 [[IV]], i64 4)
+; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i16> [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp ne <4 x i16> [[TMP7]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP12]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP12]], i1 false)
+; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[IV]], [[TMP15]]
+; CHECK-NEXT: br label %[[EARLY_EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[SPEC_LOAD_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A1:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[IV1]]
+; CHECK-NEXT: [[LD_A:%.*]] = load i16, ptr [[GEP_A1]], align 2
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[IV1]]
; CHECK-NEXT: [[LD_B:%.*]] = load i16, ptr [[GEP_B]], align 2
; CHECK-NEXT: [[CMP:%.*]] = icmp ne i16 [[LD_A]], [[LD_B]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT]], label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP1]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EARLY_EXIT]]:
-; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV1]], %[[LOOP1]] ], [ [[TMP16]], %[[VECTOR_EARLY_EXIT]] ]
; CHECK-NEXT: ret i64 [[IV_LCSSA]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i64 -1
@@ -147,21 +184,58 @@ exit:
; CHECK-LABEL: define i64 @load_i32(
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[SPEC_LOAD_CHECK:.*]]
+; CHECK: [[SPEC_LOAD_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[A]], i64 16)
+; CHECK-NEXT: [[TMP1:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[B]], i64 16)
+; CHECK-NEXT: [[TMP2:%.*]] = and i1 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = xor i1 [[TMP2]], true
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP4:%.*]] = and i64 [[N]], 3
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP4]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT: [[LD_A:%.*]] = load i32, ptr [[GEP_A]], align 4
-; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[TMP6:%.*]] = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr [[GEP_A]], i1 false, ptr @speculativeLoadOracle.1, ptr [[A]], ptr [[B]], i64 [[IV]], i64 4)
+; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i32> [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT: [[TMP9:%.*]] = call <4 x i32> (ptr, i1, ...) @llvm.speculative.load.v4i32.p0(ptr [[TMP8]], i1 false, ptr @speculativeLoadOracle.1, ptr [[A]], ptr [[B]], i64 [[IV]], i64 4)
+; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i32> [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = icmp ne <4 x i32> [[TMP7]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP12]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP13]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP12]], i1 false)
+; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[IV]], [[TMP15]]
+; CHECK-NEXT: br label %[[EARLY_EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[SPEC_LOAD_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV1]]
+; CHECK-NEXT: [[LD_A:%.*]] = load i32, ptr [[GEP_A1]], align 4
+; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV1]]
; CHECK-NEXT: [[LD_B:%.*]] = load i32, ptr [[GEP_B]], align 4
; CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[LD_A]], [[LD_B]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT]], label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP1]], label %[[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EARLY_EXIT]]:
-; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV1]], %[[LOOP1]] ], [ [[TMP16]], %[[VECTOR_EARLY_EXIT]] ]
; CHECK-NEXT: ret i64 [[IV_LCSSA]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i64 -1
@@ -170,20 +244,53 @@ exit:
; CHECK-LABEL: define i64 @load_double(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[SPEC_LOAD_CHECK:.*]]
+; CHECK: [[SPEC_LOAD_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[A]], i64 16)
+; CHECK-NEXT: [[TMP1:%.*]] = xor i1 [[TMP0]], true
+; CHECK-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[N]], 1
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP2]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[IV]]
-; CHECK-NEXT: [[LD_A:%.*]] = load double, ptr [[GEP_A]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = call <2 x double> (ptr, i1, ...) @llvm.speculative.load.v2f64.p0(ptr [[GEP_A]], i1 false, ptr @speculativeLoadOracle.2, ptr [[A]], i64 [[IV]], i64 2)
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <2 x double> [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast <2 x double> [[TMP5]] to <2 x i64>
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <2 x i64> [[TMP6]], zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = freeze <2 x i1> [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> [[TMP8]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 2
+; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP9]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> [[TMP8]], i1 false)
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[IV]], [[TMP11]]
+; CHECK-NEXT: br label %[[EARLY_EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[SPEC_LOAD_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A1:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[IV1]]
+; CHECK-NEXT: [[LD_A:%.*]] = load double, ptr [[GEP_A1]], align 8
; CHECK-NEXT: [[BITS:%.*]] = bitcast double [[LD_A]] to i64
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[BITS]], 0
-; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT]], label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP1]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[EARLY_EXIT]]:
-; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP]] ]
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV1]], %[[LOOP1]] ], [ [[TMP12]], %[[VECTOR_EARLY_EXIT]] ]
; CHECK-NEXT: ret i64 [[IV_LCSSA]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i64 -1
@@ -229,3 +336,86 @@ exit:
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i64 -1
;
+;
+; CHECK-LABEL: define internal i64 @speculativeLoadOracle(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[IV_START:%.*]], i64 [[VF:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: [[ORACLE_ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ORACLE_ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[IV_START]], [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP5:%.*]] = load i16, ptr [[TMP4]], align 2
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i16, ptr [[TMP10]], align 2
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ne i16 [[TMP5]], [[TMP8]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[ORACLE_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[VF]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[ORACLE_EXIT]], label %[[LOOP]]
+; CHECK: [[ORACLE_EXIT]]:
+; CHECK-NEXT: [[LANES:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[BYTES:%.*]] = shl i64 [[LANES]], 1
+; CHECK-NEXT: ret i64 [[BYTES]]
+;
+;
+; CHECK-LABEL: define internal i64 @speculativeLoadOracle.1(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[IV_START:%.*]], i64 [[VF:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ORACLE_ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ORACLE_ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[IV_START]], [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP10]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ne i32 [[TMP5]], [[TMP8]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[ORACLE_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[VF]]
+; CHECK-NEXT: br i1 [[TMP7]], label %[[ORACLE_EXIT]], label %[[LOOP]]
+; CHECK: [[ORACLE_EXIT]]:
+; CHECK-NEXT: [[LANES:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[BYTES:%.*]] = shl i64 [[LANES]], 2
+; CHECK-NEXT: ret i64 [[BYTES]]
+;
+;
+; CHECK-LABEL: define internal i64 @speculativeLoadOracle.2(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[IV_START:%.*]], i64 [[VF:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ORACLE_ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ORACLE_ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[IV_START]], [[INDEX]]
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP3:%.*]] = load double, ptr [[TMP8]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast double [[TMP3]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[TMP4]], 0
+; CHECK-NEXT: br i1 [[TMP5]], label %[[ORACLE_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[VF]]
+; CHECK-NEXT: br i1 [[TMP6]], label %[[ORACLE_EXIT]], label %[[LOOP]]
+; CHECK: [[ORACLE_EXIT]]:
+; CHECK-NEXT: [[LANES:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[BYTES:%.*]] = shl i64 [[LANES]], 3
+; CHECK-NEXT: ret i64 [[BYTES]]
+;
+;.
+; CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+; CHECK: attributes #[[ATTR1]] = { mustprogress noinline nosync nounwind optnone willreturn memory(argmem: read) }
+; CHECK: attributes #[[ATTR2:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: read) }
+; CHECK: attributes #[[ATTR3:[0-9]+]] = { nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) }
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-safety.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-safety.ll
index 4666532861e8c3..efbd3d30bcd3f2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-safety.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-safety.ll
@@ -1,12 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -p loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
+; RUN: opt -p loop-vectorize -enable-early-exit-vectorization-with-speculative-loads -force-vector-width=4 -S %s | FileCheck %s
target triple = "arm64-apple-macosx"
@K = internal global [4 x i8] [i8 9, i8 2, i8 9, i8 9], align 4
-; The invariant load is dereferenceable, it must be broadcast rather than
-; widened.
+; TODO: The invariant load is dereferenceable, but is not treated as safe yet.
+; It is not a consecutive access, so the loop is not vectorized.
define i64 @invariant_load(ptr %A, i64 %n) {
; CHECK-LABEL: define i64 @invariant_load(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
@@ -97,3 +97,451 @@ early.exit:
exit:
ret i64 -1
}
+
+; The vector loop may contract the fmul and fadd, the oracle may not.
+define i64 @fp_math_exit(ptr %A, i64 %n) {
+; CHECK-LABEL: define i64 @fp_math_exit(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LD_A:%.*]] = load float, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[MUL:%.*]] = fmul contract float [[LD_A]], 3.000000e+00
+; CHECK-NEXT: [[ADD:%.*]] = fadd contract float [[MUL]], 1.000000e+00
+; CHECK-NEXT: [[CONV:%.*]] = fptosi float [[ADD]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV]], 7
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT:.*]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ]
+; CHECK-NEXT: ret i64 [[IV_LCSSA]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 -1
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds float, ptr %A, i64 %iv
+ %ld.A = load float, ptr %gep.A, align 4
+ %mul = fmul contract float %ld.A, 3.0
+ %add = fadd contract float %mul, 1.0
+ %conv = fptosi float %add to i32
+ %cmp = icmp eq i32 %conv, 7
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+; The oracle's derived FP induction may differ from the vector loop's.
+define i64 @fp_induction_exit(ptr %A, i64 %n) {
+; CHECK-LABEL: define i64 @fp_induction_exit(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[F_IV:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[F_IV_NEXT:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT: [[LD_A:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT: [[BITS:%.*]] = bitcast float [[F_IV]] to i32
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[LD_A]], [[BITS]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[F_IV_NEXT]] = fadd fast float [[F_IV]], 1.000000e-01
+; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT:.*]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ]
+; CHECK-NEXT: ret i64 [[IV_LCSSA]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 -1
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %f.iv = phi float [ 0.0, %entry ], [ %f.iv.next, %loop.latch ]
+ %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv
+ %ld.A = load i32, ptr %gep.A, align 4
+ %bits = bitcast float %f.iv to i32
+ %cmp = icmp eq i32 %ld.A, %bits
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %f.iv.next = fadd fast float %f.iv, 1.0e-1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+; Null is a valid address, the oracle must not load from it.
+define i64 @null_base_null_pointer_is_valid(i64 %start, i64 %n) null_pointer_is_valid {
+; CHECK-LABEL: define i64 @null_base_null_pointer_is_valid(
+; CHECK-SAME: i64 [[START:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr null, i64 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load i8, ptr [[GEP]], align 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[LD]], 42
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT:.*]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ]
+; CHECK-NEXT: ret i64 [[IV_LCSSA]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 -1
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ %start, %entry ], [ %iv.next, %loop.latch ]
+ %gep = getelementptr i8, ptr null, i64 %iv
+ %ld = load i8, ptr %gep, align 1
+ %cmp = icmp eq i8 %ld, 42
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+; Null is a valid address in address space 1, the oracle must not load from it.
+define i64 @null_base_addrspace1(i64 %start, i64 %n) {
+; CHECK-LABEL: define i64 @null_base_addrspace1(
+; CHECK-SAME: i64 [[START:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[START]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr addrspace(1) null, i64 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load i8, ptr addrspace(1) [[GEP]], align 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[LD]], 42
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT:.*]]
+; CHECK: [[EARLY_EXIT]]:
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ]
+; CHECK-NEXT: ret i64 [[IV_LCSSA]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret i64 -1
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ %start, %entry ], [ %iv.next, %loop.latch ]
+ %gep = getelementptr i8, ptr addrspace(1) null, i64 %iv
+ %ld = load i8, ptr addrspace(1) %gep, align 1
+ %cmp = icmp eq i8 %ld, 42
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i64 %iv
+
+exit:
+ ret i64 -1
+}
+
+; The allocas are too small for the trip count, the loads need a runtime check.
+define i64 @same_exit_block_pre_inc_use1_too_small_allocas() {
+; CHECK-LABEL: define i64 @same_exit_block_pre_inc_use1_too_small_allocas() {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[P1:%.*]] = alloca [42 x i8], align 1
+; CHECK-NEXT: [[P2:%.*]] = alloca [42 x i8], align 1
+; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
+; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr nuw i8, ptr [[P2]], i64 3
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr nuw i8, ptr [[P1]], i64 3
+; CHECK-NEXT: br label %[[SPEC_LOAD_CHECK:.*]]
+; CHECK: [[SPEC_LOAD_CHECK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[TMP1]], i64 4)
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[TMP0]], i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = and i1 [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = xor i1 [[TMP4]], true
+; CHECK-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = call <4 x i8> (ptr, i1, ...) @llvm.speculative.load.v4i8.p0(ptr [[TMP7]], i1 false, ptr @speculativeLoadOracle, ptr [[P1]], ptr [[P2]], i64 [[TMP6]], i64 4)
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <4 x i8> [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP11:%.*]] = call <4 x i8> (ptr, i1, ...) @llvm.speculative.load.v4i8.p0(ptr [[TMP10]], i1 false, ptr @speculativeLoadOracle, ptr [[P1]], ptr [[P2]], i64 [[TMP6]], i64 4)
+; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i8> [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = icmp ne <4 x i8> [[TMP9]], [[TMP12]]
+; CHECK-NEXT: [[TMP14:%.*]] = freeze <4 x i1> [[TMP13]]
+; CHECK-NEXT: [[TMP15:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP14]])
+; CHECK-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 64
+; CHECK-NEXT: br i1 [[TMP15]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[LOOP_END:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP17:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP14]], i1 false)
+; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[INDEX1]], [[TMP17]]
+; CHECK-NEXT: [[TMP19:%.*]] = add i64 3, [[TMP18]]
+; CHECK-NEXT: br label %[[LOOP_END]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], %[[LOOP_INC:.*]] ], [ 3, %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
+; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
+; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
+; CHECK-NEXT: [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
+; CHECK-NEXT: br i1 [[CMP3]], label %[[LOOP_INC]], label %[[LOOP_END]]
+; CHECK: [[LOOP_INC]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP]], label %[[LOOP_END]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[LOOP_END]]:
+; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], %[[LOOP]] ], [ 67, %[[LOOP_INC]] ], [ 67, %[[MIDDLE_BLOCK]] ], [ [[TMP19]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: ret i64 [[RETVAL]]
+;
+entry:
+ %p1 = alloca [42 x i8]
+ %p2 = alloca [42 x i8]
+ call void @init_mem(ptr %p1, i64 1024)
+ call void @init_mem(ptr %p2, i64 1024)
+ br label %loop
+
+loop:
+ %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
+ %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
+ %ld1 = load i8, ptr %arrayidx, align 1
+ %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
+ %ld2 = load i8, ptr %arrayidx1, align 1
+ %cmp3 = icmp eq i8 %ld1, %ld2
+ br i1 %cmp3, label %loop.inc, label %loop.end
+
+loop.inc:
+ %index.next = add i64 %index, 1
+ %exitcond = icmp ne i64 %index.next, 67
+ br i1 %exitcond, label %loop, label %loop.end
+
+loop.end:
+ %retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
+ ret i64 %retval
+}
+
+; The dereferenceable bytes are too small for the trip count, the loads need a
+; runtime check.
+define i64 @same_exit_block_pre_inc_use1_too_small_deref_ptrs(ptr dereferenceable(42) %p1, ptr dereferenceable(42) %p2) {
+; CHECK-LABEL: define i64 @same_exit_block_pre_inc_use1_too_small_deref_ptrs(
+; CHECK-SAME: ptr dereferenceable(42) [[P1:%.*]], ptr dereferenceable(42) [[P2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr nuw i8, ptr [[P2]], i64 3
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr nuw i8, ptr [[P1]], i64 3
+; CHECK-NEXT: br label %[[SPEC_LOAD_CHECK:.*]]
+; CHECK: [[SPEC_LOAD_CHECK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[TMP1]], i64 4)
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[TMP0]], i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = and i1 [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = xor i1 [[TMP4]], true
+; CHECK-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = call <4 x i8> (ptr, i1, ...) @llvm.speculative.load.v4i8.p0(ptr [[TMP7]], i1 false, ptr @speculativeLoadOracle.1, ptr [[P1]], ptr [[P2]], i64 [[TMP6]], i64 4)
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <4 x i8> [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP11:%.*]] = call <4 x i8> (ptr, i1, ...) @llvm.speculative.load.v4i8.p0(ptr [[TMP10]], i1 false, ptr @speculativeLoadOracle.1, ptr [[P1]], ptr [[P2]], i64 [[TMP6]], i64 4)
+; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i8> [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = icmp ne <4 x i8> [[TMP9]], [[TMP12]]
+; CHECK-NEXT: [[TMP14:%.*]] = freeze <4 x i1> [[TMP13]]
+; CHECK-NEXT: [[TMP15:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP14]])
+; CHECK-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 64
+; CHECK-NEXT: br i1 [[TMP15]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[LOOP_END:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP17:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP14]], i1 false)
+; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[INDEX1]], [[TMP17]]
+; CHECK-NEXT: [[TMP19:%.*]] = add i64 3, [[TMP18]]
+; CHECK-NEXT: br label %[[LOOP_END]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], %[[LOOP_INC:.*]] ], [ 3, %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
+; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
+; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
+; CHECK-NEXT: [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
+; CHECK-NEXT: br i1 [[CMP3]], label %[[LOOP_INC]], label %[[LOOP_END]]
+; CHECK: [[LOOP_INC]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP]], label %[[LOOP_END]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK: [[LOOP_END]]:
+; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], %[[LOOP]] ], [ 67, %[[LOOP_INC]] ], [ 67, %[[MIDDLE_BLOCK]] ], [ [[TMP19]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: ret i64 [[RETVAL]]
+;
+entry:
+ br label %loop
+
+loop:
+ %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
+ %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
+ %ld1 = load i8, ptr %arrayidx, align 1
+ %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
+ %ld2 = load i8, ptr %arrayidx1, align 1
+ %cmp3 = icmp eq i8 %ld1, %ld2
+ br i1 %cmp3, label %loop.inc, label %loop.end
+
+loop.inc:
+ %index.next = add i64 %index, 1
+ %exitcond = icmp ne i64 %index.next, 67
+ br i1 %exitcond, label %loop, label %loop.end
+
+loop.end:
+ %retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
+ ret i64 %retval
+}
+
+; Dereferenceability is unknown, the loads need a runtime check.
+define i64 @same_exit_block_pre_inc_use1_unknown_ptrs(ptr %p1, ptr %p2) {
+; CHECK-LABEL: define i64 @same_exit_block_pre_inc_use1_unknown_ptrs(
+; CHECK-SAME: ptr [[P1:%.*]], ptr [[P2:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr nuw i8, ptr [[P2]], i64 3
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr nuw i8, ptr [[P1]], i64 3
+; CHECK-NEXT: br label %[[SPEC_LOAD_CHECK:.*]]
+; CHECK: [[SPEC_LOAD_CHECK]]:
+; CHECK-NEXT: [[TMP2:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[TMP1]], i64 4)
+; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[TMP0]], i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = and i1 [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = xor i1 [[TMP4]], true
+; CHECK-NEXT: br i1 [[TMP5]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 3, [[INDEX1]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = call <4 x i8> (ptr, i1, ...) @llvm.speculative.load.v4i8.p0(ptr [[TMP7]], i1 false, ptr @speculativeLoadOracle.2, ptr [[P1]], ptr [[P2]], i64 [[TMP6]], i64 4)
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <4 x i8> [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP11:%.*]] = call <4 x i8> (ptr, i1, ...) @llvm.speculative.load.v4i8.p0(ptr [[TMP10]], i1 false, ptr @speculativeLoadOracle.2, ptr [[P1]], ptr [[P2]], i64 [[TMP6]], i64 4)
+; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i8> [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = icmp ne <4 x i8> [[TMP9]], [[TMP12]]
+; CHECK-NEXT: [[TMP14:%.*]] = freeze <4 x i1> [[TMP13]]
+; CHECK-NEXT: [[TMP15:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP14]])
+; CHECK-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 4
+; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 64
+; CHECK-NEXT: br i1 [[TMP15]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[LOOP_END:.*]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP17:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP14]], i1 false)
+; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[INDEX1]], [[TMP17]]
+; CHECK-NEXT: [[TMP19:%.*]] = add i64 3, [[TMP18]]
+; CHECK-NEXT: br label %[[LOOP_END]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], %[[LOOP_INC:.*]] ], [ 3, %[[SCALAR_PH]] ]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
+; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
+; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
+; CHECK-NEXT: [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
+; CHECK-NEXT: br i1 [[CMP3]], label %[[LOOP_INC]], label %[[LOOP_END]]
+; CHECK: [[LOOP_INC]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP]], label %[[LOOP_END]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK: [[LOOP_END]]:
+; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], %[[LOOP]] ], [ 67, %[[LOOP_INC]] ], [ 67, %[[MIDDLE_BLOCK]] ], [ [[TMP19]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT: ret i64 [[RETVAL]]
+;
+entry:
+ br label %loop
+
+loop:
+ %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
+ %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
+ %ld1 = load i8, ptr %arrayidx, align 1
+ %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
+ %ld2 = load i8, ptr %arrayidx1, align 1
+ %cmp3 = icmp eq i8 %ld1, %ld2
+ br i1 %cmp3, label %loop.inc, label %loop.end
+
+loop.inc:
+ %index.next = add i64 %index, 1
+ %exitcond = icmp ne i64 %index.next, 67
+ br i1 %exitcond, label %loop, label %loop.end
+
+loop.end:
+ %retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
+ ret i64 %retval
+}
+
+declare void @init_mem(ptr, i64) nofree
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-scalable.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-scalable.ll
index 827179661f997f..d05e4ea319d6d4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-scalable.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early-exit-with-speculative-load-scalable.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --include-generated-funcs --version 6
-; RUN: opt -p loop-vectorize -mattr=+sve -S %s | FileCheck %s
+; RUN: opt -p loop-vectorize -enable-early-exit-vectorization-with-speculative-loads -mattr=+sve -S %s | FileCheck %s
target triple = "aarch64-unknown-linux-gnu"
@@ -32,23 +32,84 @@ exit:
; CHECK-LABEL: define i64 @find_first_eq_const(
; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 16
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[SPEC_LOAD_CHECK:.*]]
+; CHECK: [[SPEC_LOAD_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[A]], i64 16)
+; CHECK-NEXT: [[TMP1:%.*]] = xor i1 [[TMP0]], true
+; CHECK-NEXT: br i1 [[TMP1]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[N]], 15
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[TMP2]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = call <16 x i8> (ptr, i1, ...) @llvm.speculative.load.v16i8.p0(ptr [[TMP3]], i1 false, ptr @speculativeLoadOracle, ptr [[A]], i64 [[INDEX]], i64 16)
+; CHECK-NEXT: [[TMP5:%.*]] = freeze <16 x i8> [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <16 x i8> [[TMP5]], splat (i8 42)
+; CHECK-NEXT: [[TMP7:%.*]] = freeze <16 x i1> [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP7]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> [[TMP7]], i1 false)
+; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]]
+; CHECK-NEXT: br label %[[EARLY_EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[SPEC_LOAD_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[LD_A]], 42
-; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[EARLY_EXIT]], label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EARLY_EXIT]]:
-; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ]
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ], [ [[TMP11]], %[[VECTOR_EARLY_EXIT]] ]
; CHECK-NEXT: ret i64 [[IV_LCSSA]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i64 -1
;
+;
+; CHECK-LABEL: define internal i64 @speculativeLoadOracle(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[IV_START:%.*]], i64 [[VF:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-NEXT: [[ORACLE_ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
+; CHECK: [[LOOP_HEADER]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ORACLE_ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[IV_START]], [[INDEX]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i8 [[TMP5]], 42
+; CHECK-NEXT: br i1 [[TMP6]], label %[[ORACLE_EXIT:.*]], label %[[LOOP_LATCH]]
+; CHECK: [[LOOP_LATCH]]:
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[VF]]
+; CHECK-NEXT: br i1 [[TMP3]], label %[[ORACLE_EXIT]], label %[[LOOP_HEADER]]
+; CHECK: [[ORACLE_EXIT]]:
+; CHECK-NEXT: [[LANES:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT: ret i64 [[LANES]]
+;
;.
; CHECK: attributes #[[ATTR0]] = { "target-features"="+sve" }
+; CHECK: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+; CHECK: attributes #[[ATTR2]] = { mustprogress noinline nosync nounwind optnone willreturn memory(argmem: read) }
+; CHECK: attributes #[[ATTR3:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: read) }
+; CHECK: attributes #[[ATTR4:[0-9]+]] = { nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) }
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_cost.ll
index 5879557b7c3a12..8c02c2da1a175a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/early_exit_cost.ll
@@ -1,48 +1,114 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -p loop-vectorize -S %s | FileCheck %s
-; RUN: opt -p loop-vectorize -vectorizer-maximize-bandwidth -S %s | FileCheck --check-prefix=MAX-BW %s
+; RUN: opt -p loop-vectorize -enable-early-exit-vectorization-with-speculative-loads -S %s | FileCheck %s
+; RUN: opt -p loop-vectorize -enable-early-exit-vectorization-with-speculative-loads -vectorizer-maximize-bandwidth -S %s | FileCheck --check-prefix=MAX-BW %s
target triple = "arm64-apple-macosx"
define i64 @early_exit_with_without_dereferenceable(ptr %p1, ptr %p2) {
; CHECK-LABEL: define i64 @early_exit_with_without_dereferenceable(
; CHECK-SAME: ptr [[P1:%.*]], ptr [[P2:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
-; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[SPEC_LOAD_CHECK:.*]]
+; CHECK: [[SPEC_LOAD_CHECK]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[P1]], i64 16)
+; CHECK-NEXT: [[TMP1:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[P2]], i64 16)
+; CHECK-NEXT: [[TMP2:%.*]] = and i1 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = xor i1 [[TMP2]], true
+; CHECK-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[GEP_P1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
-; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[GEP_P1]], align 1
-; CHECK-NEXT: [[GEP_P2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
+; CHECK-NEXT: [[TMP5:%.*]] = call <16 x i8> (ptr, i1, ...) @llvm.speculative.load.v16i8.p0(ptr [[GEP_P1]], i1 false, ptr @speculativeLoadOracle, ptr [[P1]], ptr [[P2]], i64 [[IV]], i64 16)
+; CHECK-NEXT: [[TMP6:%.*]] = freeze <16 x i8> [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
+; CHECK-NEXT: [[TMP8:%.*]] = call <16 x i8> (ptr, i1, ...) @llvm.speculative.load.v16i8.p0(ptr [[TMP7]], i1 false, ptr @speculativeLoadOracle, ptr [[P1]], ptr [[P2]], i64 [[IV]], i64 16)
+; CHECK-NEXT: [[TMP9:%.*]] = freeze <16 x i8> [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = icmp ne <16 x i8> [[TMP6]], [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = freeze <16 x i1> [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP11]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 16
+; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
+; CHECK-NEXT: br i1 [[TMP12]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; CHECK: [[VECTOR_BODY_INTERIM]]:
+; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: br label %[[SCALAR_PH]]
+; CHECK: [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> [[TMP11]], i1 false)
+; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[IV]], [[TMP14]]
+; CHECK-NEXT: br label %[[EXIT:.*]]
+; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[MIDDLE_BLOCK]] ], [ 0, %[[SPEC_LOAD_CHECK]] ]
+; CHECK-NEXT: br label %[[LOOP_HEADER1:.*]]
+; CHECK: [[LOOP_HEADER1]]:
+; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[GEP_P3:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV1]]
+; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[GEP_P3]], align 1
+; CHECK-NEXT: [[GEP_P2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV1]]
; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[GEP_P2]], align 1
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
-; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP_LATCH]], label %[[EXIT:.*]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP_LATCH]], label %[[EXIT]]
; CHECK: [[LOOP_LATCH]]:
-; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[IV_NEXT]], 100
-; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP_HEADER]], label %[[EXIT]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP_HEADER1]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP_LATCH]] ], [ [[IV]], %[[LOOP_HEADER]] ]
+; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV1]], %[[LOOP_LATCH]] ], [ [[IV1]], %[[LOOP_HEADER1]] ], [ [[TMP15]], %[[VECTOR_EARLY_EXIT]] ]
; CHECK-NEXT: ret i64 [[IV_LCSSA]]
;
; MAX-BW-LABEL: define i64 @early_exit_with_without_dereferenceable(
; MAX-BW-SAME: ptr [[P1:%.*]], ptr [[P2:%.*]]) {
-; MAX-BW-NEXT: [[ENTRY:.*]]:
-; MAX-BW-NEXT: br label %[[LOOP_HEADER:.*]]
-; MAX-BW: [[LOOP_HEADER]]:
-; MAX-BW-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; MAX-BW-NEXT: [[ENTRY:.*:]]
+; MAX-BW-NEXT: br label %[[SPEC_LOAD_CHECK:.*]]
+; MAX-BW: [[SPEC_LOAD_CHECK]]:
+; MAX-BW-NEXT: [[TMP0:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[P1]], i64 16)
+; MAX-BW-NEXT: [[TMP1:%.*]] = call i1 @llvm.can.load.speculatively.p0(ptr [[P2]], i64 16)
+; MAX-BW-NEXT: [[TMP2:%.*]] = and i1 [[TMP0]], [[TMP1]]
+; MAX-BW-NEXT: [[TMP3:%.*]] = xor i1 [[TMP2]], true
+; MAX-BW-NEXT: br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; MAX-BW: [[VECTOR_PH]]:
+; MAX-BW-NEXT: br label %[[VECTOR_BODY:.*]]
+; MAX-BW: [[VECTOR_BODY]]:
+; MAX-BW-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; MAX-BW-NEXT: [[GEP_P1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
-; MAX-BW-NEXT: [[LD1:%.*]] = load i8, ptr [[GEP_P1]], align 1
-; MAX-BW-NEXT: [[GEP_P2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
+; MAX-BW-NEXT: [[TMP5:%.*]] = call <16 x i8> (ptr, i1, ...) @llvm.speculative.load.v16i8.p0(ptr [[GEP_P1]], i1 false, ptr @speculativeLoadOracle, ptr [[P1]], ptr [[P2]], i64 [[IV]], i64 16)
+; MAX-BW-NEXT: [[TMP6:%.*]] = freeze <16 x i8> [[TMP5]]
+; MAX-BW-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
+; MAX-BW-NEXT: [[TMP8:%.*]] = call <16 x i8> (ptr, i1, ...) @llvm.speculative.load.v16i8.p0(ptr [[TMP7]], i1 false, ptr @speculativeLoadOracle, ptr [[P1]], ptr [[P2]], i64 [[IV]], i64 16)
+; MAX-BW-NEXT: [[TMP9:%.*]] = freeze <16 x i8> [[TMP8]]
+; MAX-BW-NEXT: [[TMP10:%.*]] = icmp ne <16 x i8> [[TMP6]], [[TMP9]]
+; MAX-BW-NEXT: [[TMP11:%.*]] = freeze <16 x i1> [[TMP10]]
+; MAX-BW-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP11]])
+; MAX-BW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 16
+; MAX-BW-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
+; MAX-BW-NEXT: br i1 [[TMP12]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
+; MAX-BW: [[VECTOR_BODY_INTERIM]]:
+; MAX-BW-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; MAX-BW: [[MIDDLE_BLOCK]]:
+; MAX-BW-NEXT: br label %[[SCALAR_PH]]
+; MAX-BW: [[VECTOR_EARLY_EXIT]]:
+; MAX-BW-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> [[TMP11]], i1 false)
+; MAX-BW-NEXT: [[TMP15:%.*]] = add i64 [[IV]], [[TMP14]]
+; MAX-BW-NEXT: br label %[[EXIT:.*]]
+; MAX-BW: [[SCALAR_PH]]:
+; MAX-BW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[MIDDLE_BLOCK]] ], [ 0, %[[SPEC_LOAD_CHECK]] ]
+; MAX-BW-NEXT: br label %[[LOOP_HEADER1:.*]]
+; MAX-BW: [[LOOP_HEADER1]]:
+; MAX-BW-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; MAX-BW-NEXT: [[GEP_P3:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV1]]
+; MAX-BW-NEXT: [[LD1:%.*]] = load i8, ptr [[GEP_P3]], align 1
+; MAX-BW-NEXT: [[GEP_P2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV1]]
; MAX-BW-NEXT: [[LD2:%.*]] = load i8, ptr [[GEP_P2]], align 1
; MAX-BW-NEXT: [[CMP:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
-; MAX-BW-NEXT: br i1 [[CMP]], label %[[LOOP_LATCH]], label %[[EXIT:.*]]
+; MAX-BW-NEXT: br i1 [[CMP]], label %[[LOOP_LATCH]], label %[[EXIT]]
; MAX-BW: [[LOOP_LATCH]]:
-; MAX-BW-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; MAX-BW-NEXT: [[IV_NEXT]] = add i64 [[IV1]], 1
; MAX-BW-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[IV_NEXT]], 100
-; MAX-BW-NEXT: br i1 [[EXITCOND]], label %[[LOOP_HEADER]], label %[[EXIT]]
+; MAX-BW-NEXT: br i1 [[EXITCOND]], label %[[LOOP_HEADER1]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
; MAX-BW: [[EXIT]]:
-; MAX-BW-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP_LATCH]] ], [ [[IV]], %[[LOOP_HEADER]] ]
+; MAX-BW-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV1]], %[[LOOP_LATCH]] ], [ [[IV1]], %[[LOOP_HEADER1]] ], [ [[TMP15]], %[[VECTOR_EARLY_EXIT]] ]
; MAX-BW-NEXT: ret i64 [[IV_LCSSA]]
;
entry:
@@ -130,3 +196,14 @@ loop.latch:
exit:
ret i64 %iv
}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
+; MAX-BW: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; MAX-BW: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; MAX-BW: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; MAX-BW: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/early-exit-with-speculative-load-oracle.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/early-exit-with-speculative-load-oracle.ll
index 7ae491ba18af8b..ea31e33baacd86 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/early-exit-with-speculative-load-oracle.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/early-exit-with-speculative-load-oracle.ll
@@ -1,12 +1,101 @@
-; RUN: opt -p loop-vectorize -force-vector-width=4 -disable-output \
-; RUN: -vplan-print-after=printOptimizedVPlan %s 2>&1 | FileCheck %s --allow-empty
-; CHECK-NOT: VPlan for loop in
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -p loop-vectorize -enable-early-exit-vectorization-with-speculative-loads -force-vector-width=4 -disable-output \
+; RUN: -vplan-print-after=printOptimizedVPlan %s 2>&1 | FileCheck %s
target triple = "arm64-apple-macosx"
@G = external global [1024 x i8]
define i64 @find_first_eq_const(ptr %A, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'find_first_eq_const'
+; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: SPECULATIVE-LOAD-ORACLE vp<[[VP3:%[0-9]+]]>
+; CHECK: VPlan 'Speculative-load oracle for VF={1},UF>=1' {
+; CHECK-EMPTY:
+; CHECK-NEXT: oracle.entry:
+; CHECK-NEXT: EMIT-SCALAR vp<%A> = live-in
+; CHECK-NEXT: EMIT-SCALAR vp<%iv.start> = live-in
+; CHECK-NEXT: EMIT-SCALAR vp<%vf> = live-in
+; CHECK-NEXT: Successor(s): loop.header
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.header:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, oracle.entry ], [ vp<%index.next>, loop.latch ]
+; CHECK-NEXT: EMIT vp<[[VP1]]> = add vp<%iv.start>, vp<%index>
+; CHECK-NEXT: CLONE ir<%gep.A> = getelementptr inbounds vp<%A>, vp<[[VP1]]>
+; CHECK-NEXT: CLONE ir<%ld.A> = load ir<%gep.A>
+; CHECK-NEXT: EMIT ir<%cmp> = icmp eq ir<%ld.A>, ir<42>
+; CHECK-NEXT: EMIT branch-on-cond ir<%cmp> (!vplan.prof.estimated estimated {67108864, 2080374784})
+; CHECK-NEXT: Successor(s): oracle.exit, loop.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.latch:
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP2]]> = icmp eq vp<%index.next>, vp<%vf>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP2]]>
+; CHECK-NEXT: Successor(s): oracle.exit, loop.header
+; CHECK-EMPTY:
+; CHECK-NEXT: oracle.exit:
+; CHECK-NEXT: EMIT vp<%lanes> = add vp<%index>, ir<1>
+; CHECK-NEXT: EMIT ret vp<%lanes>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-EMPTY:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%gep.A> = getelementptr inbounds ir<%A>, vp<[[VP5]]>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP6:%[0-9]+]]> = call llvm.speculative.load(ir<%gep.A>, ir<false>, vp<[[VP3]]>, ir<%A>, vp<[[VP5]]>, vp<[[VP0]]>)
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = freeze vp<[[VP6]]>
+; CHECK-NEXT: WIDEN ir<%cmp> = icmp eq vp<[[VP7]]>, ir<42>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = freeze ir<%cmp>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): vector.early.exit, middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.early.exit:
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = first-active-lane vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = add vp<[[VP4]]>, vp<[[VP13]]>
+; CHECK-NEXT: Successor(s): ir-bb<early.exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<early.exit>:
+; CHECK-NEXT: IR %iv.lcssa = phi i64 [ %iv, %loop.header ] (extra operand: vp<[[VP14]]> from vector.early.exit)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<loop.header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop.header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
+; CHECK-NEXT: IR %ld.A = load i8, ptr %gep.A, align 1
+; CHECK-NEXT: IR %cmp = icmp eq i8 %ld.A, 42
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
;
entry:
br label %loop.header
@@ -31,6 +120,96 @@ exit:
}
define i64 @find_first_eq_live_in(ptr %A, i64 %n, i8 %val) {
+; CHECK-LABEL: VPlan for loop in 'find_first_eq_live_in'
+; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: SPECULATIVE-LOAD-ORACLE vp<[[VP3:%[0-9]+]]>
+; CHECK: VPlan 'Speculative-load oracle for VF={1},UF>=1' {
+; CHECK-EMPTY:
+; CHECK-NEXT: oracle.entry:
+; CHECK-NEXT: EMIT-SCALAR vp<%A> = live-in
+; CHECK-NEXT: EMIT-SCALAR vp<%val> = live-in
+; CHECK-NEXT: EMIT-SCALAR vp<%iv.start> = live-in
+; CHECK-NEXT: EMIT-SCALAR vp<%vf> = live-in
+; CHECK-NEXT: Successor(s): loop.header
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.header:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, oracle.entry ], [ vp<%index.next>, loop.latch ]
+; CHECK-NEXT: EMIT vp<[[VP1]]> = add vp<%iv.start>, vp<%index>
+; CHECK-NEXT: CLONE ir<%gep.A> = getelementptr inbounds vp<%A>, vp<[[VP1]]>
+; CHECK-NEXT: CLONE ir<%ld.A> = load ir<%gep.A>
+; CHECK-NEXT: EMIT ir<%cmp> = icmp eq ir<%ld.A>, vp<%val>
+; CHECK-NEXT: EMIT branch-on-cond ir<%cmp> (!vplan.prof.estimated estimated {67108864, 2080374784})
+; CHECK-NEXT: Successor(s): oracle.exit, loop.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.latch:
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP2]]> = icmp eq vp<%index.next>, vp<%vf>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP2]]>
+; CHECK-NEXT: Successor(s): oracle.exit, loop.header
+; CHECK-EMPTY:
+; CHECK-NEXT: oracle.exit:
+; CHECK-NEXT: EMIT vp<%lanes> = add vp<%index>, ir<1>
+; CHECK-NEXT: EMIT ret vp<%lanes>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-EMPTY:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%gep.A> = getelementptr inbounds ir<%A>, vp<[[VP5]]>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP6:%[0-9]+]]> = call llvm.speculative.load(ir<%gep.A>, ir<false>, vp<[[VP3]]>, ir<%A>, ir<%val>, vp<[[VP5]]>, vp<[[VP0]]>)
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = freeze vp<[[VP6]]>
+; CHECK-NEXT: WIDEN ir<%cmp> = icmp eq vp<[[VP7]]>, ir<%val>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = freeze ir<%cmp>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): vector.early.exit, middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.early.exit:
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = first-active-lane vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = add vp<[[VP4]]>, vp<[[VP13]]>
+; CHECK-NEXT: Successor(s): ir-bb<early.exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<early.exit>:
+; CHECK-NEXT: IR %iv.lcssa = phi i64 [ %iv, %loop.header ] (extra operand: vp<[[VP14]]> from vector.early.exit)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<loop.header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop.header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
+; CHECK-NEXT: IR %ld.A = load i8, ptr %gep.A, align 1
+; CHECK-NEXT: IR %cmp = icmp eq i8 %ld.A, %val
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
;
entry:
br label %loop.header
@@ -55,6 +234,105 @@ exit:
}
define i32 @i32_induction(ptr %A, ptr %B, i32 %n) {
+; CHECK-LABEL: VPlan for loop in 'i32_induction'
+; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: SPECULATIVE-LOAD-ORACLE vp<[[VP3:%[0-9]+]]>
+; CHECK: VPlan 'Speculative-load oracle for VF={1},UF>=1' {
+; CHECK-EMPTY:
+; CHECK-NEXT: oracle.entry:
+; CHECK-NEXT: EMIT-SCALAR vp<%A> = live-in
+; CHECK-NEXT: EMIT-SCALAR vp<%B> = live-in
+; CHECK-NEXT: EMIT-SCALAR vp<%iv.start> = live-in
+; CHECK-NEXT: EMIT-SCALAR vp<%vf> = live-in
+; CHECK-NEXT: Successor(s): loop.header
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.header:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, oracle.entry ], [ vp<%index.next>, loop.latch ]
+; CHECK-NEXT: EMIT vp<[[VP1]]> = add vp<%iv.start>, vp<%index>
+; CHECK-NEXT: CLONE ir<%gep.A> = getelementptr inbounds vp<%A>, vp<[[VP1]]>
+; CHECK-NEXT: CLONE ir<%ld.A> = load ir<%gep.A>
+; CHECK-NEXT: CLONE ir<%gep.B> = getelementptr inbounds vp<%B>, vp<[[VP1]]>
+; CHECK-NEXT: CLONE ir<%ld.B> = load ir<%gep.B>
+; CHECK-NEXT: EMIT ir<%cmp> = icmp ne ir<%ld.A>, ir<%ld.B>
+; CHECK-NEXT: EMIT branch-on-cond ir<%cmp> (!vplan.prof.estimated estimated {67108864, 2080374784})
+; CHECK-NEXT: Successor(s): oracle.exit, loop.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.latch:
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP2]]> = icmp eq vp<%index.next>, vp<%vf>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP2]]>
+; CHECK-NEXT: Successor(s): oracle.exit, loop.header
+; CHECK-EMPTY:
+; CHECK-NEXT: oracle.exit:
+; CHECK-NEXT: EMIT vp<%lanes> = add vp<%index>, ir<1>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP4:%[0-9]+]]> = zext vp<%lanes> to i64
+; CHECK-NEXT: EMIT ret vp<[[VP4]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-EMPTY:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%gep.A> = getelementptr inbounds ir<%A>, vp<[[VP5]]>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP6:%[0-9]+]]> = call llvm.speculative.load(ir<%gep.A>, ir<false>, vp<[[VP3]]>, ir<%A>, ir<%B>, vp<[[VP5]]>, vp<[[VP0]]>)
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = freeze vp<[[VP6]]>
+; CHECK-NEXT: CLONE ir<%gep.B> = getelementptr inbounds ir<%B>, vp<[[VP5]]>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP8:%[0-9]+]]> = call llvm.speculative.load(ir<%gep.B>, ir<false>, vp<[[VP3]]>, ir<%A>, ir<%B>, vp<[[VP5]]>, vp<[[VP0]]>)
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = freeze vp<[[VP8]]>
+; CHECK-NEXT: WIDEN ir<%cmp> = icmp ne vp<[[VP7]]>, vp<[[VP9]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = freeze ir<%cmp>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = any-of vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP11]]>, vp<[[VP12]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): vector.early.exit, middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.early.exit:
+; CHECK-NEXT: EMIT vp<[[VP15:%[0-9]+]]> = first-active-lane vp<[[VP10]]>
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP16:%[0-9]+]]> = trunc vp<[[VP15]]> to i32
+; CHECK-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = add vp<[[VP4]]>, vp<[[VP16]]>
+; CHECK-NEXT: Successor(s): ir-bb<early.exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<early.exit>:
+; CHECK-NEXT: IR %iv.lcssa = phi i32 [ %iv, %loop.header ] (extra operand: vp<[[VP17]]> from vector.early.exit)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<loop.header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop.header>:
+; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %gep.A = getelementptr inbounds i8, ptr %A, i32 %iv
+; CHECK-NEXT: IR %ld.A = load i8, ptr %gep.A, align 1
+; CHECK-NEXT: IR %gep.B = getelementptr inbounds i8, ptr %B, i32 %iv
+; CHECK-NEXT: IR %ld.B = load i8, ptr %gep.B, align 1
+; CHECK-NEXT: IR %cmp = icmp ne i8 %ld.A, %ld.B
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
;
entry:
br label %loop.header
@@ -80,9 +358,9 @@ exit:
ret i32 -1
}
-; Covers replaying scalar binops, div/rem with a constant divisor and freeze.
+; The oracle cannot replay freeze independently of the vector loop.
+; CHECK-NOT: VPlan for loop in 'binop_chain'
define i64 @binop_chain(ptr %A, ptr %B, ptr %C, i64 %n) {
-;
entry:
br label %loop.header
@@ -186,6 +464,107 @@ exit:
}
define i64 @derived_induction_start(ptr %A, ptr %B, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'derived_induction_start'
+; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<10> + vp<[[VP2]]> * ir<1>
+; CHECK-NEXT: SPECULATIVE-LOAD-ORACLE vp<[[VP4:%[0-9]+]]>
+; CHECK: VPlan 'Speculative-load oracle for VF={1},UF>=1' {
+; CHECK-EMPTY:
+; CHECK-NEXT: oracle.entry:
+; CHECK-NEXT: EMIT-SCALAR vp<%A> = live-in
+; CHECK-NEXT: EMIT-SCALAR vp<%B> = live-in
+; CHECK-NEXT: EMIT-SCALAR vp<%iv.start> = live-in
+; CHECK-NEXT: EMIT-SCALAR vp<%vf> = live-in
+; CHECK-NEXT: Successor(s): loop.header
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.header:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, oracle.entry ], [ vp<%index.next>, loop.latch ]
+; CHECK-NEXT: EMIT vp<[[VP1]]> = add vp<%iv.start>, vp<%index>
+; CHECK-NEXT: CLONE ir<%gep.A> = getelementptr inbounds vp<%A>, vp<[[VP1]]>
+; CHECK-NEXT: CLONE ir<%ld.A> = load ir<%gep.A>
+; CHECK-NEXT: CLONE ir<%gep.B> = getelementptr inbounds vp<%B>, vp<[[VP1]]>
+; CHECK-NEXT: CLONE ir<%ld.B> = load ir<%gep.B>
+; CHECK-NEXT: EMIT ir<%cmp> = icmp ne ir<%ld.A>, ir<%ld.B>
+; CHECK-NEXT: EMIT branch-on-cond ir<%cmp> (!vplan.prof.estimated estimated {67108864, 2080374784})
+; CHECK-NEXT: Successor(s): oracle.exit, loop.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.latch:
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP2]]> = icmp eq vp<%index.next>, vp<%vf>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP2]]>
+; CHECK-NEXT: Successor(s): oracle.exit, loop.header
+; CHECK-EMPTY:
+; CHECK-NEXT: oracle.exit:
+; CHECK-NEXT: EMIT vp<%lanes> = add vp<%index>, ir<1>
+; CHECK-NEXT: EMIT ret vp<%lanes>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-EMPTY:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = DERIVED-IV nuw ir<10> + vp<[[VP5]]> * ir<1>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = SCALAR-STEPS vp<[[VP6]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%gep.A> = getelementptr inbounds ir<%A>, vp<[[VP7]]>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP8:%[0-9]+]]> = call llvm.speculative.load(ir<%gep.A>, ir<false>, vp<[[VP4]]>, ir<%A>, ir<%B>, vp<[[VP7]]>, vp<[[VP0]]>)
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = freeze vp<[[VP8]]>
+; CHECK-NEXT: CLONE ir<%gep.B> = getelementptr inbounds ir<%B>, vp<[[VP7]]>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP10:%[0-9]+]]> = call llvm.speculative.load(ir<%gep.B>, ir<false>, vp<[[VP4]]>, ir<%A>, ir<%B>, vp<[[VP7]]>, vp<[[VP0]]>)
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = freeze vp<[[VP10]]>
+; CHECK-NEXT: WIDEN ir<%cmp> = icmp ne vp<[[VP9]]>, vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = freeze ir<%cmp>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = any-of vp<[[VP12]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP13]]>, vp<[[VP14]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): vector.early.exit, middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.early.exit:
+; CHECK-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = first-active-lane vp<[[VP12]]>
+; CHECK-NEXT: EMIT vp<[[VP18:%[0-9]+]]> = add vp<[[VP5]]>, vp<[[VP17]]>
+; CHECK-NEXT: Successor(s): ir-bb<early.exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<early.exit>:
+; CHECK-NEXT: IR %iv.lcssa = phi i64 [ %iv, %loop.header ] (extra operand: vp<[[VP18]]> from vector.early.exit)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val>.1 = phi [ vp<[[VP3]]>, middle.block ], [ ir<10>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<loop.header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop.header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.off = phi i64 [ 10, %entry ], [ %iv.off.next, %loop.latch ] (extra operand: vp<%bc.resume.val>.1 from scalar.ph)
+; CHECK-NEXT: IR %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv.off
+; CHECK-NEXT: IR %ld.A = load i8, ptr %gep.A, align 1
+; CHECK-NEXT: IR %gep.B = getelementptr inbounds i8, ptr %B, i64 %iv.off
+; CHECK-NEXT: IR %ld.B = load i8, ptr %gep.B, align 1
+; CHECK-NEXT: IR %cmp = icmp ne i8 %ld.A, %ld.B
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
;
entry:
br label %loop.header
@@ -215,6 +594,106 @@ exit:
; Test with induction with step 2.
define i64 @derived_induction_step(ptr %A, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'derived_induction_step'
+; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = DERIVED-IV ir<0> + vp<[[VP2]]> * ir<2>
+; CHECK-NEXT: SPECULATIVE-LOAD-ORACLE vp<[[VP4:%[0-9]+]]>
+; CHECK: VPlan 'Speculative-load oracle for VF={1},UF>=1' {
+; CHECK-EMPTY:
+; CHECK-NEXT: oracle.entry:
+; CHECK-NEXT: EMIT-SCALAR vp<%A> = live-in
+; CHECK-NEXT: EMIT-SCALAR vp<%iv.start> = live-in
+; CHECK-NEXT: EMIT-SCALAR vp<%iv.start>.1 = live-in
+; CHECK-NEXT: EMIT-SCALAR vp<%vf> = live-in
+; CHECK-NEXT: Successor(s): loop.header
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.header:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, oracle.entry ], [ vp<%index.next>, loop.latch ]
+; CHECK-NEXT: EMIT vp<[[VP1]]> = add vp<%iv.start>, vp<%index>
+; CHECK-NEXT: EMIT vp<[[VP2]]> = shl vp<%index>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP3]]> = add vp<%iv.start>.1, vp<[[VP2]]>
+; CHECK-NEXT: CLONE ir<%gep.A> = getelementptr inbounds vp<%A>, vp<[[VP1]]>
+; CHECK-NEXT: CLONE ir<%ld.A> = load ir<%gep.A>
+; CHECK-NEXT: EMIT-SCALAR ir<%trunc> = trunc vp<[[VP3]]> to i8
+; CHECK-NEXT: EMIT ir<%cmp> = icmp ne ir<%ld.A>, ir<%trunc>
+; CHECK-NEXT: EMIT branch-on-cond ir<%cmp> (!vplan.prof.estimated estimated {67108864, 2080374784})
+; CHECK-NEXT: Successor(s): oracle.exit, loop.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.latch:
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP4]]> = icmp eq vp<%index.next>, vp<%vf>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP4]]>
+; CHECK-NEXT: Successor(s): oracle.exit, loop.header
+; CHECK-EMPTY:
+; CHECK-NEXT: oracle.exit:
+; CHECK-NEXT: EMIT vp<%lanes> = add vp<%index>, ir<1>
+; CHECK-NEXT: EMIT ret vp<%lanes>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-EMPTY:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: ir<%iv.2> = WIDEN-INDUCTION ir<0>, ir<2>, vp<[[VP0]]> (truncated to i8)
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = SCALAR-STEPS vp<[[VP5]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = DERIVED-IV nuw ir<0> + vp<[[VP5]]> * ir<2>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = SCALAR-STEPS vp<[[VP7]]>, ir<2>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%gep.A> = getelementptr inbounds ir<%A>, vp<[[VP6]]>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP9:%[0-9]+]]> = call llvm.speculative.load(ir<%gep.A>, ir<false>, vp<[[VP4]]>, ir<%A>, vp<[[VP6]]>, vp<[[VP8]]>, vp<[[VP0]]>)
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = freeze vp<[[VP9]]>
+; CHECK-NEXT: WIDEN ir<%cmp> = icmp ne vp<[[VP10]]>, ir<%iv.2>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = freeze ir<%cmp>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = any-of vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP12]]>, vp<[[VP13]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): vector.early.exit, middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.early.exit:
+; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = first-active-lane vp<[[VP11]]>
+; CHECK-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = add vp<[[VP5]]>, vp<[[VP16]]>
+; CHECK-NEXT: Successor(s): ir-bb<early.exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<early.exit>:
+; CHECK-NEXT: IR %iv.lcssa = phi i64 [ %iv, %loop.header ] (extra operand: vp<[[VP17]]> from vector.early.exit)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val>.1 = phi [ vp<[[VP3]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<loop.header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop.header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %iv.2 = phi i64 [ 0, %entry ], [ %iv.2.next, %loop.latch ] (extra operand: vp<%bc.resume.val>.1 from scalar.ph)
+; CHECK-NEXT: IR %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
+; CHECK-NEXT: IR %ld.A = load i8, ptr %gep.A, align 1
+; CHECK-NEXT: IR %trunc = trunc i64 %iv.2 to i8
+; CHECK-NEXT: IR %cmp = icmp ne i8 %ld.A, %trunc
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
;
entry:
br label %loop.header
@@ -243,6 +722,97 @@ exit:
; Dead recipes are dropped before the oracle plan is built.
define i64 @dead_recipes_in_loop(ptr %A, i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'dead_recipes_in_loop'
+; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: SPECULATIVE-LOAD-ORACLE vp<[[VP3:%[0-9]+]]>
+; CHECK: VPlan 'Speculative-load oracle for VF={1},UF>=1' {
+; CHECK-EMPTY:
+; CHECK-NEXT: oracle.entry:
+; CHECK-NEXT: EMIT-SCALAR vp<%A> = live-in
+; CHECK-NEXT: EMIT-SCALAR vp<%iv.start> = live-in
+; CHECK-NEXT: EMIT-SCALAR vp<%vf> = live-in
+; CHECK-NEXT: Successor(s): loop.header
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.header:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, oracle.entry ], [ vp<%index.next>, loop.latch ]
+; CHECK-NEXT: EMIT vp<[[VP1]]> = add vp<%iv.start>, vp<%index>
+; CHECK-NEXT: CLONE ir<%gep.A> = getelementptr inbounds vp<%A>, vp<[[VP1]]>
+; CHECK-NEXT: CLONE ir<%ld.A> = load ir<%gep.A>
+; CHECK-NEXT: EMIT ir<%cmp> = icmp eq ir<%ld.A>, ir<42>
+; CHECK-NEXT: EMIT branch-on-cond ir<%cmp> (!vplan.prof.estimated estimated {67108864, 2080374784})
+; CHECK-NEXT: Successor(s): oracle.exit, loop.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.latch:
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP2]]> = icmp eq vp<%index.next>, vp<%vf>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP2]]>
+; CHECK-NEXT: Successor(s): oracle.exit, loop.header
+; CHECK-EMPTY:
+; CHECK-NEXT: oracle.exit:
+; CHECK-NEXT: EMIT vp<%lanes> = add vp<%index>, ir<1>
+; CHECK-NEXT: EMIT ret vp<%lanes>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-EMPTY:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%gep.A> = getelementptr inbounds ir<%A>, vp<[[VP5]]>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP6:%[0-9]+]]> = call llvm.speculative.load(ir<%gep.A>, ir<false>, vp<[[VP3]]>, ir<%A>, vp<[[VP5]]>, vp<[[VP0]]>)
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = freeze vp<[[VP6]]>
+; CHECK-NEXT: WIDEN ir<%cmp> = icmp eq vp<[[VP7]]>, ir<42>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = freeze ir<%cmp>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): vector.early.exit, middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.early.exit:
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = first-active-lane vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = add vp<[[VP4]]>, vp<[[VP13]]>
+; CHECK-NEXT: Successor(s): ir-bb<early.exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<early.exit>:
+; CHECK-NEXT: IR %iv.lcssa = phi i64 [ %iv, %loop.header ] (extra operand: vp<[[VP14]]> from vector.early.exit)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<loop.header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop.header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %dead.call = call i8 @llvm.abs.i8(i8 0, i1 false)
+; CHECK-NEXT: IR %dead.gep = getelementptr i8, ptr null, i64 8
+; CHECK-NEXT: IR %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
+; CHECK-NEXT: IR %ld.A = load i8, ptr %gep.A, align 1
+; CHECK-NEXT: IR %cmp = icmp eq i8 %ld.A, 42
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
;
entry:
br label %loop.header
@@ -270,6 +840,95 @@ exit:
; The oracle only reads memory through its arguments, so @G is passed in.
define i64 @global_base(i64 %n) {
+; CHECK-LABEL: VPlan for loop in 'global_base'
+; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: SPECULATIVE-LOAD-ORACLE vp<[[VP3:%[0-9]+]]>
+; CHECK: VPlan 'Speculative-load oracle for VF={1},UF>=1' {
+; CHECK-EMPTY:
+; CHECK-NEXT: oracle.entry:
+; CHECK-NEXT: EMIT-SCALAR vp<%G> = live-in
+; CHECK-NEXT: EMIT-SCALAR vp<%iv.start> = live-in
+; CHECK-NEXT: EMIT-SCALAR vp<%vf> = live-in
+; CHECK-NEXT: Successor(s): loop.header
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.header:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, oracle.entry ], [ vp<%index.next>, loop.latch ]
+; CHECK-NEXT: EMIT vp<[[VP1]]> = add vp<%iv.start>, vp<%index>
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds vp<%G>, vp<[[VP1]]>
+; CHECK-NEXT: CLONE ir<%l> = load ir<%gep>
+; CHECK-NEXT: EMIT ir<%c> = icmp eq ir<%l>, ir<42>
+; CHECK-NEXT: EMIT branch-on-cond ir<%c> (!vplan.prof.estimated estimated {67108864, 2080374784})
+; CHECK-NEXT: Successor(s): oracle.exit, loop.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.latch:
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP2]]> = icmp eq vp<%index.next>, vp<%vf>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP2]]>
+; CHECK-NEXT: Successor(s): oracle.exit, loop.header
+; CHECK-EMPTY:
+; CHECK-NEXT: oracle.exit:
+; CHECK-NEXT: EMIT vp<%lanes> = add vp<%index>, ir<1>
+; CHECK-NEXT: EMIT ret vp<%lanes>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-EMPTY:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<@G>, vp<[[VP5]]>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP6:%[0-9]+]]> = call llvm.speculative.load(ir<%gep>, ir<false>, vp<[[VP3]]>, ir<@G>, vp<[[VP5]]>, vp<[[VP0]]>)
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = freeze vp<[[VP6]]>
+; CHECK-NEXT: WIDEN ir<%c> = icmp eq vp<[[VP7]]>, ir<42>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = freeze ir<%c>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): vector.early.exit, middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.early.exit:
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = first-active-lane vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP14:%[0-9]+]]> = add vp<[[VP4]]>, vp<[[VP13]]>
+; CHECK-NEXT: Successor(s): ir-bb<early.exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<early.exit>:
+; CHECK-NEXT: IR %iv.lcssa = phi i64 [ %iv, %loop.header ] (extra operand: vp<[[VP14]]> from vector.early.exit)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<loop.header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop.header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %gep = getelementptr inbounds i8, ptr @G, i64 %iv
+; CHECK-NEXT: IR %l = load i8, ptr %gep, align 1
+; CHECK-NEXT: IR %c = icmp eq i8 %l, 42
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
;
entry:
br label %loop.header
@@ -417,3 +1076,120 @@ early.exit:
exit:
ret i64 -1
}
+
+; An exit live-out may use %scale, but the oracle only needs the exit test.
+define i8 @liveout_only(ptr %A, i64 %n, i8 %scale) {
+; CHECK-LABEL: VPlan for loop in 'liveout_only'
+; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: SPECULATIVE-LOAD-ORACLE vp<[[VP3:%[0-9]+]]>
+; CHECK: VPlan 'Speculative-load oracle for VF={1},UF>=1' {
+; CHECK-EMPTY:
+; CHECK-NEXT: oracle.entry:
+; CHECK-NEXT: EMIT-SCALAR vp<%A> = live-in
+; CHECK-NEXT: EMIT-SCALAR vp<%iv.start> = live-in
+; CHECK-NEXT: EMIT-SCALAR vp<%vf> = live-in
+; CHECK-NEXT: Successor(s): loop.header
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.header:
+; CHECK-NEXT: EMIT-SCALAR vp<%index> = phi [ ir<0>, oracle.entry ], [ vp<%index.next>, loop.latch ]
+; CHECK-NEXT: EMIT vp<[[VP1]]> = add vp<%iv.start>, vp<%index>
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds vp<%A>, vp<[[VP1]]>
+; CHECK-NEXT: CLONE ir<%ld> = load ir<%gep>
+; CHECK-NEXT: EMIT ir<%cmp> = icmp eq ir<%ld>, ir<42>
+; CHECK-NEXT: EMIT branch-on-cond ir<%cmp> (!vplan.prof.estimated estimated {67108864, 2080374784})
+; CHECK-NEXT: Successor(s): oracle.exit, loop.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: loop.latch:
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%index>, ir<1>
+; CHECK-NEXT: EMIT vp<[[VP2]]> = icmp eq vp<%index.next>, vp<%vf>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP2]]>
+; CHECK-NEXT: Successor(s): oracle.exit, loop.header
+; CHECK-EMPTY:
+; CHECK-NEXT: oracle.exit:
+; CHECK-NEXT: EMIT vp<%lanes> = add vp<%index>, ir<1>
+; CHECK-NEXT: EMIT ret vp<%lanes>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-EMPTY:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = SCALAR-STEPS vp<[[VP4]]>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%A>, vp<[[VP5]]>
+; CHECK-NEXT: WIDEN-INTRINSIC vp<[[VP6:%[0-9]+]]> = call llvm.speculative.load(ir<%gep>, ir<false>, vp<[[VP3]]>, ir<%A>, vp<[[VP5]]>, vp<[[VP0]]>)
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = freeze vp<[[VP6]]>
+; CHECK-NEXT: WIDEN ir<%cmp> = icmp eq vp<[[VP7]]>, ir<42>
+; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = freeze ir<%cmp>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = any-of vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = icmp eq vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-two-conds vp<[[VP9]]>, vp<[[VP10]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): vector.early.exit, middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<[[VP2]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.early.exit:
+; CHECK-NEXT: WIDEN ir<%val> = add vp<[[VP7]]>, ir<%scale>
+; CHECK-NEXT: EMIT vp<%first.active.lane> = first-active-lane vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<%early.exit.value> = extract-lane vp<%first.active.lane>, ir<%val>
+; CHECK-NEXT: Successor(s): ir-bb<early.exit>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<early.exit>:
+; CHECK-NEXT: IR %val.lcssa = phi i8 [ %val, %loop.header ] (extra operand: vp<%early.exit.value> from vector.early.exit)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP2]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<loop.header>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop.header>:
+; CHECK-NEXT: IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp<%bc.resume.val> from scalar.ph)
+; CHECK-NEXT: IR %gep = getelementptr inbounds i8, ptr %A, i64 %iv
+; CHECK-NEXT: IR %ld = load i8, ptr %gep, align 1
+; CHECK-NEXT: IR %val = add i8 %ld, %scale
+; CHECK-NEXT: IR %cmp = icmp eq i8 %ld, 42
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop.header
+
+loop.header:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+ %gep = getelementptr inbounds i8, ptr %A, i64 %iv
+ %ld = load i8, ptr %gep, align 1
+ %val = add i8 %ld, %scale
+ %cmp = icmp eq i8 %ld, 42
+ br i1 %cmp, label %early.exit, label %loop.latch
+
+loop.latch:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %ec = icmp ne i64 %iv.next, %n
+ br i1 %ec, label %loop.header, label %exit
+
+early.exit:
+ ret i8 %val
+
+exit:
+ ret i8 0
+}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
index 5fd844e186e44e..373230ba25cd4f 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-before-after-all.ll
@@ -75,6 +75,7 @@
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::materializePacksAndUnpacks
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::materializeBroadcasts
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::replicateByVF
+; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::attachSpeculativeLoadChecks
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::materializeConstantVectorTripCount
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::optimizeForVFAndUF
; CHECK: VPlan for loop in 'foo' [[BEFORE_OR_AFTER]] VPlanTransforms::combineRecipes at 2
diff --git a/llvm/test/Transforms/LoopVectorize/unsupported_early_exit.ll b/llvm/test/Transforms/LoopVectorize/unsupported_early_exit.ll
index a9393d6b6a570b..d2308b72bd639d 100644
--- a/llvm/test/Transforms/LoopVectorize/unsupported_early_exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/unsupported_early_exit.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 | FileCheck %s
+; RUN: opt -S < %s -p loop-vectorize -enable-early-exit-vectorization-with-speculative-loads -force-vector-width=4 | FileCheck %s
declare void @init_mem(ptr, i64);
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
index 86dc6da2aa5149..4bcb2d016dc7c9 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
@@ -23,6 +23,68 @@ LLVM_ABI extern cl::opt<bool> VerifyEachVPlan;
using VPVerifierTest = VPlanTestBase;
namespace {
+TEST_F(VPVerifierTest, ReturnTerminator) {
+ VPlan &Plan = getPlan();
+ auto *Exit = Plan.createVPBasicBlock("exit");
+ Plan.setEntry(Exit);
+ VPValue *One = Plan.getConstantInt(64, 1);
+ VPBuilder Builder(Exit);
+ Builder.createNaryOp(Instruction::Ret, One);
+ EXPECT_TRUE(verifyVPlanIsValid(Plan));
+
+ auto CheckInvalidReturn = [&]() {
+#if GTEST_HAS_STREAM_REDIRECTION
+ ::testing::internal::CaptureStderr();
+#endif
+ EXPECT_FALSE(verifyVPlanIsValid(Plan));
+#if GTEST_HAS_STREAM_REDIRECTION
+ EXPECT_STREQ(
+ "Return must terminate a top-level block without successors!\n",
+ ::testing::internal::GetCapturedStderr().c_str());
+#endif
+ };
+
+ // A return must be the last recipe in its block.
+ VPInstruction *AfterRet = Builder.createAdd(One, One);
+ CheckInvalidReturn();
+ AfterRet->eraseFromParent();
+
+ // A return cannot have a successor.
+ VPBlockUtils::connectBlocks(Exit, Plan.getScalarHeader());
+ CheckInvalidReturn();
+ VPBlockUtils::disconnectBlocks(Exit, Plan.getScalarHeader());
+
+ // A return cannot terminate a block inside a loop region.
+ auto *Entry = Plan.createVPBasicBlock("entry");
+ Plan.setEntry(Entry);
+ auto *Region = Plan.createLoopRegion(One->getScalarType(), DebugLoc(), "loop",
+ Exit, Exit);
+ VPBlockUtils::connectBlocks(Entry, Region);
+ CheckInvalidReturn();
+}
+
+TEST_F(VPVerifierTest, LiveIn) {
+ VPlan &Plan = getPlan();
+ VPBasicBlock *Entry = Plan.getEntry();
+ Type *I64Ty = Type::getInt64Ty(C);
+ VPBuilder Builder(Entry);
+ VPInstruction *LiveIn = Builder.createLiveIn(I64Ty);
+ EXPECT_TRUE(verifyVPlanIsValid(Plan));
+
+ // Live-ins must be in the plan's entry block.
+ VPBasicBlock *VPBB = Plan.createVPBasicBlock("bb");
+ VPBlockUtils::connectBlocks(Entry, VPBB);
+ LiveIn->moveBefore(*VPBB, VPBB->end());
+#if GTEST_HAS_STREAM_REDIRECTION
+ ::testing::internal::CaptureStderr();
+#endif
+ EXPECT_FALSE(verifyVPlanIsValid(Plan));
+#if GTEST_HAS_STREAM_REDIRECTION
+ EXPECT_STREQ("Live-in must be in the plan's entry block!\n",
+ ::testing::internal::GetCapturedStderr().c_str());
+#endif
+}
+
TEST_F(VPVerifierTest, VPInstructionUseBeforeDefSameBB) {
VPlan &Plan = getPlan();
VPIRValue *Zero = Plan.getConstantInt(32, 0);
More information about the llvm-commits
mailing list