[llvm-branch-commits] [llvm] [VPlan] Implement VPlan-based unit-strideness speculation (PR #182595)
Andrei Elovikov via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Jul 17 09:45:12 PDT 2026
https://github.com/eas updated https://github.com/llvm/llvm-project/pull/182595
>From 193fc77621feef3dfa7bede8ddb650e51661050c Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Wed, 25 Feb 2026 10:01:33 -0800
Subject: [PATCH 1/3] [VPlan] Implement VPlan-based stride speculation
---
.../Vectorize/LoopVectorizationPlanner.h | 6 +
.../Transforms/Vectorize/LoopVectorize.cpp | 10 +
.../Transforms/Vectorize/VPRecipeBuilder.h | 12 +-
llvm/lib/Transforms/Vectorize/VPlan.cpp | 4 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 42 +
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 7 +
.../Transforms/Vectorize/VPlanTransforms.cpp | 251 ++-
.../Transforms/Vectorize/VPlanTransforms.h | 6 +
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 5 +-
.../VPlan/vplan-based-stride-mv.ll | 621 +++---
.../LoopVectorize/vplan-based-stride-mv.ll | 1672 ++++++++---------
11 files changed, 1478 insertions(+), 1158 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index cd02ced2942d1..f6c5da8126c7f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -516,6 +516,12 @@ class VPBuilder {
return tryInsertInstruction(new VPExpandSCEVRecipe(Expr));
}
+ VPExpandStridePredicatesRecipe *
+ createExpandSCEVPredicate(const SCEVUnionPredicate &StridePredicates) {
+ return tryInsertInstruction(new VPExpandStridePredicatesRecipe(
+ StridePredicates, this->getPlan().getContext()));
+ }
+
VPVectorPointerRecipe *
createVectorPointer(VPValue *Ptr, Type *SourceElementTy, VPValue *Stride,
GEPNoWrapFlags GEPFlags, DebugLoc DL) {
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1929f52ae7d95..118d4402cbf37 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6375,6 +6375,16 @@ bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
return false;
}
+bool VPRecipeBuilder::isConsecutiveWithoutVPlanBasedStrideSpeculation(
+ VPInstruction *MemOp) {
+ auto *I = MemOp->getUnderlyingInstr();
+ auto *PtrOp = getLoadStorePointerOperand(I);
+ auto *ScalarTy = MemOp->getOpcode() == Instruction::Load
+ ? I->getType()
+ : I->getOperand(0)->getType();
+ return Legal->isConsecutivePtr(ScalarTy, PtrOp);
+}
+
VPSingleDefRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
VFRange &Range) {
auto *I = VPI->getUnderlyingInstr();
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 1303b62d5faf4..0281192d68f3b 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -33,11 +33,6 @@ class VPRecipeBuilder {
VPBuilder &Builder;
- /// Check if \p I can be widened at the start of \p Range and possibly
- /// decrease the range such that the returned value holds for the entire \p
- /// Range. The function should not be called for memory instructions or calls.
- bool shouldWiden(Instruction *I, VFRange &Range) const;
-
/// Optimize the special case where the operand of \p VPI is a constant
/// integer induction variable.
VPWidenIntOrFpInductionRecipe *
@@ -61,6 +56,11 @@ class VPRecipeBuilder {
/// Returns true if the target prefers vectorized addressing.
bool prefersVectorizedAddressing() const;
+ /// Check if \p I can be widened at the start of \p Range and possibly
+ /// decrease the range such that the returned value holds for the entire \p
+ /// Range. The function should not be called for memory instructions or calls.
+ bool shouldWiden(Instruction *I, VFRange &Range) const;
+
/// Create and return a widened recipe for a non-phi recipe \p R if one can be
/// created within the given VF \p Range.
VPRecipeBase *tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
@@ -90,6 +90,8 @@ class VPRecipeBuilder {
/// predicated, add the mask as last operand. Range.End may be decreased to
/// ensure same recipe behavior from \p Range.Start to \p Range.End.
VPSingleDefRecipe *handleReplication(VPInstruction *VPI, VFRange &Range);
+
+ bool isConsecutiveWithoutVPlanBasedStrideSpeculation(VPInstruction *MemOp);
};
} // end namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 7d69b3453cba9..10f957609f238 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -677,7 +677,9 @@ void VPBasicBlock::print(raw_ostream &O, const Twine &Indent,
auto RecipeIndent = Indent + " ";
for (const VPRecipeBase &Recipe : *this) {
Recipe.print(O, RecipeIndent, SlotTracker);
- O << '\n';
+ // SCEVPredicate::print adds a newline so we don't want to add one for it:
+ if (!isa<VPExpandStridePredicatesRecipe>(Recipe))
+ O << '\n';
}
printSuccessors(O, Indent);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b67d1a1281c14..64ab410a77b53 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -418,6 +418,7 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
VPBranchOnMaskSC,
VPDerivedIVSC,
VPExpandSCEVSC,
+ VPExpandStridePredicatesSC,
VPExpressionSC,
VPIRInstructionSC,
VPInstructionSC,
@@ -625,6 +626,7 @@ class LLVM_ABI_FOR_TEST VPSingleDefRecipe : public VPRecipeBase,
switch (R->getVPRecipeID()) {
case VPRecipeBase::VPDerivedIVSC:
case VPRecipeBase::VPExpandSCEVSC:
+ case VPRecipeBase::VPExpandStridePredicatesSC:
case VPRecipeBase::VPExpressionSC:
case VPRecipeBase::VPInstructionSC:
case VPRecipeBase::VPReductionEVLSC:
@@ -4021,6 +4023,46 @@ class VPExpandSCEVRecipe : public VPSingleDefRecipe {
#endif
};
+class VPExpandStridePredicatesRecipe : public VPSingleDefRecipe {
+ SCEVUnionPredicate StridePredicates;
+
+public:
+ VPExpandStridePredicatesRecipe(const SCEVUnionPredicate &StridePredicates,
+ LLVMContext &Ctx)
+ : VPSingleDefRecipe(VPRecipeBase::VPExpandStridePredicatesSC, {},
+ Type::getInt1Ty(Ctx)),
+ StridePredicates(StridePredicates) {}
+
+ ~VPExpandStridePredicatesRecipe() override = default;
+
+ VPExpandStridePredicatesRecipe *clone() override {
+ return new VPExpandStridePredicatesRecipe(StridePredicates,
+ getScalarType()->getContext());
+ }
+
+ VP_CLASSOF_IMPL(VPRecipeBase::VPExpandStridePredicatesSC)
+
+ void execute(VPTransformState &State) override {
+ llvm_unreachable("SCEVPredicates must be expanded before final execute");
+ }
+
+ /// Return the cost of this VPExpandSCEVRecipe.
+ InstructionCost computeCost(ElementCount VF,
+ VPCostContext &Ctx) const override {
+ // TODO: Compute accurate cost after retiring the legacy cost model.
+ return 0;
+ }
+
+ const SCEVPredicate *getSCEVPredicate() const { return &StridePredicates; }
+
+protected:
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ /// Print the recipe.
+ void printRecipe(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
+#endif
+};
+
/// A recipe for generating the active lane mask for the vector loop that is
/// used to predicate the vector operations.
class VPActiveLaneMaskPHIRecipe : public VPHeaderPHIRecipe {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index c62c4a082b95e..75fb9b3734e94 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -4824,6 +4824,13 @@ void VPExpandSCEVRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
printAsOperand(O, SlotTracker);
O << " = EXPAND SCEV " << *Expr;
}
+
+void VPExpandStridePredicatesRecipe::printRecipe(
+ raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const {
+ O << Indent << "EMIT ";
+ printAsOperand(O, SlotTracker);
+ O << " = EXPAND SCEVPredicate " << StridePredicates;
+}
#endif
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 1dc13889af4d6..6d556a3bd29be 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -48,6 +48,11 @@ using namespace llvm;
using namespace VPlanPatternMatch;
using namespace SCEVPatternMatch;
+static cl::opt<bool> EnableVPlanBasedStrideMV(
+ "enable-vplan-based-stride-mv", cl::init(false), cl::Hidden,
+ cl::desc("Perform stride multiversioning directly on VPlan instead of in "
+ "LoopAccessAnalysis."));
+
bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
VPlan &Plan, const TargetLibraryInfo &TLI) {
@@ -5830,6 +5835,17 @@ VPlanTransforms::expandSCEVs(VPlan &Plan, ScalarEvolution &SE) {
DenseMap<const SCEV *, Value *> ExpandedSCEVs;
// Expand remaining VPExpandSCEVRecipes to IR instructions using SCEVExpander.
for (VPRecipeBase &R : make_early_inc_range(*Entry)) {
+ if (auto *ExpStrides = dyn_cast<VPExpandStridePredicatesRecipe>(&R)) {
+ Value *Res = Expander.expandCodeForPredicate(
+ ExpStrides->getSCEVPredicate(), EntryBB->getTerminator());
+ Res->setName("strides.mv.check");
+ VPValue *Exp = Plan.getOrAddLiveIn(Res);
+
+ ExpStrides->replaceAllUsesWith(Exp);
+ ExpStrides->eraseFromParent();
+ continue;
+ }
+
auto *ExpSCEV = dyn_cast<VPExpandSCEVRecipe>(&R);
if (!ExpSCEV)
continue;
@@ -5843,8 +5859,10 @@ VPlanTransforms::expandSCEVs(VPlan &Plan, ScalarEvolution &SE) {
Plan.resetTripCount(Exp);
ExpSCEV->eraseFromParent();
}
- assert(none_of(*Entry, IsaPred<VPExpandSCEVRecipe>) &&
- "all VPExpandSCEVRecipes must have been expanded");
+ assert(none_of(*Entry,
+ IsaPred<VPExpandSCEVRecipe, VPExpandStridePredicatesRecipe>) &&
+ "all VPExpandSCEVRecipes/VPExpandStridePredicatesRecipe must have "
+ "been expanded");
// Add IR instructions in the entry basic block but not in the VPIRBasicBlock
// to the VPIRBasicBlock.
auto EI = Entry->begin();
@@ -7474,6 +7492,10 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
return ReplaceWith(VPI, WidenedR);
});
+ if (EnableVPlanBasedStrideMV)
+ RUN_VPLAN_PASS(VPlanTransforms::multiversionForUnitStridedMemOps, Plan,
+ CostCtx, RecipeBuilder, Range, MemOps);
+
VPlanTransforms::runPass("delegateMemOpWideningToLegacyCM", ProcessSubset,
Plan, [&](VPInstruction *VPI) {
if (VPRecipeBase *Recipe =
@@ -7484,6 +7506,231 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
});
}
+void VPlanTransforms::multiversionForUnitStridedMemOps(
+ VPlan &Plan, VPCostContext &CostCtx, VPRecipeBuilder &RecipeBuilder,
+ VFRange &Range, SmallVectorImpl<VPInstruction *> &MemOps) {
+ if (CostCtx.L->getHeader()->getParent()->hasOptSize())
+ return;
+ SmallVector<VPInstruction *> RemainingOps;
+
+ ScalarEvolution *SE = CostCtx.PSE.getSE();
+
+ PredicatedScalarEvolution StrideMVPSE(*SE, const_cast<Loop &>(*CostCtx.L));
+
+ SCEVUnionPredicate StridePredicates({}, *SE);
+
+ // Use `for_each` so that we could do `return Skip();`.
+ for_each(MemOps, [&](VPInstruction *VPI) {
+ auto Skip = [&]() { RemainingOps.push_back(VPI); };
+ if (RecipeBuilder.isConsecutiveWithoutVPlanBasedStrideSpeculation(VPI))
+ return Skip();
+ auto *PtrOp = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(0)
+ : VPI->getOperand(1);
+
+ const SCEV *PtrSCEV =
+ vputils::getSCEVExprForVPValue(PtrOp, CostCtx.PSE, CostCtx.L);
+ const SCEV *Start = nullptr;
+ const SCEV *Stride = nullptr;
+
+ if (!match(PtrSCEV, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Stride),
+ m_SpecificLoop(CostCtx.L)))) {
+ return Skip();
+ }
+
+ Type *ScalarTy = VPI->getOpcode() == Instruction::Load
+ ? VPI->getScalarType()
+ : VPI->getOperand(0)->getScalarType();
+
+ if (VPI->getMask()) {
+ auto &TTI = CostCtx.TTI;
+ Instruction *I = VPI->getUnderlyingInstr();
+ unsigned AS = getLoadStoreAddressSpace(I);
+ const Align Alignment = getLoadStoreAlignment(I);
+ if (!LoopVectorizationPlanner::getDecisionAndClampRange(
+ [&](ElementCount VF) -> bool {
+ Type *VTy = VectorType::get(ScalarTy, VF);
+ return VPI->getOpcode() == Instruction::Load
+ ? (TTI.isLegalMaskedLoad(VTy, Alignment, AS) ||
+ TTI.isLegalMaskedGather(VTy, Alignment))
+ : (TTI.isLegalMaskedStore(VTy, Alignment, AS) ||
+ TTI.isLegalMaskedScatter(VTy, Alignment));
+ },
+ Range))
+ return Skip();
+ }
+
+ const auto *TypeSize = cast<SCEVConstant>(SE->getSizeOfExpr(
+ Stride->getType(), SE->getDataLayout().getTypeStoreSize(ScalarTy)));
+
+ auto ReplaceWithUnitStrided = [&]() {
+ VPBuilder Builder(VPI);
+ VPValue *StrideInBytes = Plan.getOrAddLiveIn(TypeSize->getValue());
+ auto *VecPtr =
+ new VPVectorPointerRecipe(PtrOp, ScalarTy, StrideInBytes,
+ GEPNoWrapFlags::none(), VPI->getDebugLoc());
+ Builder.insert(VecPtr);
+ if (VPI->getOpcode() == Instruction::Load) {
+ auto *WideLoad = new VPWidenLoadRecipe(
+ cast<LoadInst>(*VPI->getUnderlyingInstr()), VecPtr, VPI->getMask(),
+ true, *VPI, VPI->getDebugLoc());
+ Builder.insert(WideLoad);
+ VPI->replaceAllUsesWith(WideLoad);
+ } else {
+ auto *WideStore = new VPWidenStoreRecipe(
+ cast<StoreInst>(*VPI->getUnderlyingInstr()), VecPtr,
+ VPI->getOperand(0), VPI->getMask(), true, *VPI, VPI->getDebugLoc());
+ Builder.insert(WideStore);
+ }
+ VPI->eraseFromParent();
+ };
+
+ if (isa<SCEVConstant>(Stride)) {
+ if (Stride != TypeSize)
+ return Skip();
+
+ // Earlier MV helped with this memory operation too.
+ ReplaceWithUnitStrided();
+ return;
+ }
+
+ const SCEVConstant *StrideConstantMultiplier;
+ const SCEV *StrideNonConstantMultiplier;
+
+ const SCEV *ToMultiVersion = Stride;
+ const SCEV *MVConst = TypeSize;
+ if (match(Stride, m_scev_c_Mul(m_SCEVConstant(StrideConstantMultiplier),
+ m_SCEV(StrideNonConstantMultiplier)))) {
+ if (TypeSize != StrideConstantMultiplier) {
+ // TODO: Support `TypeSize = N * StrideCosntantMultiplier`,
+ // including negative `N`. For now, only process when they're equal,
+ // which matches the usefull part of the legacy behavior that
+ // multiversiones GEP index for stride one.
+ return Skip();
+ }
+ ToMultiVersion = StrideNonConstantMultiplier;
+ MVConst = SE->getOne(ToMultiVersion->getType());
+ } else if (!TypeSize->isOne()) {
+ // Likewise - try to match legacy behavior.
+ return Skip();
+ }
+
+ while (auto *C = dyn_cast<SCEVIntegralCastExpr>(ToMultiVersion)) {
+ ToMultiVersion = C->getOperand();
+ MVConst = SE->getTruncateOrSignExtend(MVConst, ToMultiVersion->getType());
+ }
+
+ if (!isa<SCEVUnknown>(ToMultiVersion)) {
+ // Match legacy behavior.
+ // If/when changed, make sure that explicit poison/undef in the defining
+ // expression doesn't cause any issues.
+ return Skip();
+ }
+
+ Value *StrideVal = cast<SCEVUnknown>(ToMultiVersion)->getValue();
+
+ if (isa<UndefValue>(StrideVal))
+ return Skip();
+
+ const SCEVPredicate *NewPred =
+ SE->getComparePredicate(CmpInst::ICMP_EQ, ToMultiVersion, MVConst);
+
+ // Check if new predicate implies that backedge is never taken. If so, there
+ // is no reason to multiversion for it.
+ SmallVector<const SCEVPredicate *> Preds{&CostCtx.PSE.getPredicate(),
+ &StridePredicates, NewPred};
+ auto *PredicatedMaxBTC = SE->rewriteUsingPredicate(
+ SE->getSymbolicMaxBackedgeTakenCount(CostCtx.L), CostCtx.L,
+ StridePredicates.getUnionWith(NewPred, *SE)
+ .getUnionWith(&CostCtx.PSE.getPredicate(), *SE));
+
+ if (LoopVectorizationPlanner::getDecisionAndClampRange(
+ [&](ElementCount VF) {
+ return SE->isKnownPositive(SE->getMinusSCEV(
+ SE->getConstant(PredicatedMaxBTC->getType(),
+ VF.isScalable() ? 1 : VF.getFixedValue() - 1),
+ PredicatedMaxBTC));
+ },
+ Range))
+ return Skip();
+
+ StridePredicates = StridePredicates.getUnionWith(NewPred, *SE);
+
+ auto ReplaceMVUses = [&](Value *V) {
+ VPValue *From = Plan.getLiveIn(V);
+ if (!From)
+ return;
+ VPValue *To = Plan.getConstantInt(
+ From->getScalarType(),
+ cast<SCEVConstant>(MVConst)->getAPInt().getLimitedValue());
+ // TODO: Why is "If" necessary?
+ From->replaceUsesWithIf(To, [&](VPUser &U, unsigned) {
+ auto *R = cast<VPRecipeBase>(&U);
+ return R->getRegion() ||
+ R->getParent() ==
+ Plan.getVectorLoopRegion()->getSinglePredecessor();
+ });
+ };
+
+ ReplaceMVUses(StrideVal);
+ for (auto *U : StrideVal->users())
+ if (isa<SExtInst, ZExtInst>(U))
+ ReplaceMVUses(U);
+
+ ReplaceWithUnitStrided();
+ });
+
+ MemOps.swap(RemainingOps);
+
+ if (StridePredicates.isAlwaysTrue())
+ return;
+
+ VPBasicBlock *Entry = Plan.getEntry();
+ VPBuilder Builder(Entry);
+
+ auto *Pred = Builder.createExpandSCEVPredicate(StridePredicates);
+
+ auto *StridesCheckBB = Plan.createVPBasicBlock("strides.check");
+ VPBlockBase *ScalarPH = Plan.getScalarPreheader();
+ VPBlockUtils::insertBlockBefore(StridesCheckBB, Plan.getVectorPreheader());
+ VPBlockUtils::connectBlocks(StridesCheckBB, ScalarPH);
+ // SCEVExpander::expandCodeForPredicate would negate the condition, so scalar
+ // preheader should be the first successor.
+ std::swap(StridesCheckBB->getSuccessors()[0],
+ StridesCheckBB->getSuccessors()[1]);
+ Builder.setInsertPoint(StridesCheckBB);
+ Builder.createNaryOp(VPInstruction::BranchOnCond, Pred);
+
+ for (VPRecipeBase &R : cast<VPBasicBlock>(ScalarPH)->phis()) {
+ auto &Phi = cast<VPPhi>(R);
+ Phi.addIncoming(Phi.getIncomingValueForBlock(Entry));
+ }
+
+ auto RewriteVPExpandSCEV = [&](VPExpandSCEVRecipe *R) {
+ const SCEV *S = R->getSCEV();
+ Builder.setInsertPoint(R);
+ const SCEV *NewS =
+ SE->rewriteUsingPredicate(S, CostCtx.L, StridePredicates);
+ if (NewS == S)
+ return;
+ auto *NewR = Builder.createExpandSCEV(NewS);
+ R->replaceAllUsesWith(NewR);
+
+ // If this recipe is a trip count then we need to reset it explicitly.
+ if (R == Plan.getTripCount())
+ Plan.resetTripCount(NewR);
+
+ R->eraseFromParent();
+ };
+
+ if (auto *R = dyn_cast<VPExpandSCEVRecipe>(Plan.getTripCount())) {
+ RewriteVPExpandSCEV(R);
+ }
+
+ for (auto &R : make_early_inc_range(*Entry))
+ if (auto *ExpandSCEV = dyn_cast<VPExpandSCEVRecipe>(&R))
+ RewriteVPExpandSCEV(ExpandSCEV);
+}
+
void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
if (LoopVectorizationPlanner::getDecisionAndClampRange(
[&](ElementCount VF) { return VF.isScalar(); }, Range))
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 85375625d34b5..5fe168e2dcc4c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -585,6 +585,12 @@ struct VPlanTransforms {
VPRecipeBuilder &RecipeBuilder,
VPCostContext &CostCtx);
+ /// \p MemOps must be updated to contain ones that haven't been processed by
+ /// the pass.
+ static void multiversionForUnitStridedMemOps(
+ VPlan &Plan, VPCostContext &CostCtx, VPRecipeBuilder &RecipeBuilder,
+ VFRange &Range, SmallVectorImpl<VPInstruction *> &MemOps);
+
/// Make VPlan-based scalarization decision prior to delegating to the ones
/// made by the legacy CM. Only transforms "usesFirstLaneOnly` def-use chains
/// enabled by prior widening of consecutive memory operations for now.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 03e1ead89c169..98994291779b1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -426,8 +426,9 @@ bool vputils::isSingleScalar(const VPValue *VPV) {
if (auto *Expr = dyn_cast<VPExpressionRecipe>(VPV))
return Expr->isVectorToScalar();
- // VPExpandSCEVRecipes must be placed in the entry and are always uniform.
- return isa<VPExpandSCEVRecipe>(VPV);
+ // VPExpandSCEVRecipes and VPExpandStridePredicatesRecipe must be placed in
+ // the entry and are always uniform.
+ return isa<VPExpandSCEVRecipe, VPExpandStridePredicatesRecipe>(VPV);
}
bool vputils::isUniformAcrossVFsAndUFs(const VPValue *V) {
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
index db4d2ce2738a2..206dceb238d3d 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-based-stride-mv.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -disable-output \
-; RUN: -vplan-print-after=scalarizeMemOpsWithIrregularTypes \
-; RUN: -enable-mem-access-versioning=false 2>&1 | FileCheck %s
+; RUN: -vplan-print-after=multiversionForUnitStridedMemOps \
+; RUN: -enable-mem-access-versioning=false -enable-vplan-based-stride-mv 2>&1 | FileCheck %s
define void @basic(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-LABEL: VPlan for loop in 'basic'
@@ -12,31 +12,38 @@ define void @basic(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -45,7 +52,7 @@ define void @basic(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -106,7 +113,8 @@ define void @basic_optsize(ptr noalias %p.out, ptr %p, i64 %stride) #0 {
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -115,7 +123,7 @@ define void @basic_optsize(ptr noalias %p.out, ptr %p, i64 %stride) #0 {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -124,7 +132,7 @@ define void @basic_optsize(ptr noalias %p.out, ptr %p, i64 %stride) #0 {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -187,7 +195,8 @@ define void @basic_minsize(ptr noalias %p.out, ptr %p, i64 %stride) #1 {
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -196,7 +205,7 @@ define void @basic_minsize(ptr noalias %p.out, ptr %p, i64 %stride) #1 {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -205,7 +214,7 @@ define void @basic_minsize(ptr noalias %p.out, ptr %p, i64 %stride) #1 {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -257,32 +266,39 @@ define void @byte_gep_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%stride.x8> = mul ir<%stride>, ir<8>
+; CHECK-NEXT: EMIT ir<%stride.x8> = mul ir<1>, ir<8>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride.x8>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -291,7 +307,7 @@ define void @byte_gep_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -357,7 +373,8 @@ define void @byte_gep_under_scaled_stride(ptr noalias %p.out, ptr %p, i64 %strid
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -366,7 +383,7 @@ define void @byte_gep_under_scaled_stride(ptr noalias %p.out, ptr %p, i64 %strid
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -375,7 +392,7 @@ define void @byte_gep_under_scaled_stride(ptr noalias %p.out, ptr %p, i64 %strid
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -442,7 +459,8 @@ define void @byte_gep_over_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -451,7 +469,7 @@ define void @byte_gep_over_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -460,7 +478,7 @@ define void @byte_gep_over_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -526,7 +544,8 @@ define void @byte_gep_non_power_of_two_scaled_stride(ptr noalias %p.out, ptr %p,
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -535,7 +554,7 @@ define void @byte_gep_non_power_of_two_scaled_stride(ptr noalias %p.out, ptr %p,
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -544,7 +563,7 @@ define void @byte_gep_non_power_of_two_scaled_stride(ptr noalias %p.out, ptr %p,
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -608,7 +627,8 @@ define void @byte_gep_nonscaled_stride(ptr noalias %p.out, ptr %p, i64 %stride)
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -617,7 +637,7 @@ define void @byte_gep_nonscaled_stride(ptr noalias %p.out, ptr %p, i64 %stride)
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -626,7 +646,7 @@ define void @byte_gep_nonscaled_stride(ptr noalias %p.out, ptr %p, i64 %stride)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -691,7 +711,8 @@ define void @byte_gep_negated_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -700,7 +721,7 @@ define void @byte_gep_negated_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -709,7 +730,7 @@ define void @byte_gep_negated_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -758,34 +779,42 @@ define void @shared_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p0>, ir<%idx>
; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%p1>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld0> = load ir<%gep.ld0>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld1> = load ir<%gep.ld1>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld0>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld0> = load vp<[[VP6]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld1>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld1> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -794,7 +823,7 @@ define void @shared_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -847,35 +876,44 @@ define void @independent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stri
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride0 == 1
+; CHECK-NEXT: Equal predicate: %stride1 == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx0> = mul ir<%iv>, ir<%stride0>
-; CHECK-NEXT: EMIT ir<%idx1> = mul ir<%iv>, ir<%stride1>
+; CHECK-NEXT: EMIT ir<%idx0> = mul ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx1> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p0>, ir<%idx0>
; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%p1>, ir<%idx1>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld0> = load ir<%gep.ld0>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld1> = load ir<%gep.ld1>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld0>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld0> = load vp<[[VP6]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld1>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld1> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -884,7 +922,7 @@ define void @independent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stri
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -939,36 +977,43 @@ define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%stride1> = add ir<%stride>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx0> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%stride1> = add ir<1>, ir<1>
+; CHECK-NEXT: EMIT ir<%idx0> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx1> = mul ir<%iv>, ir<%stride1>
; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p0>, ir<%idx0>
; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%p1>, ir<%idx1>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld0> = load ir<%gep.ld0>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld0>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld0> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT-SCALAR ir<%ld1> = load ir<%gep.ld1>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -977,7 +1022,7 @@ define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1036,36 +1081,43 @@ define void @dependent_strides_reverse_order(ptr noalias %p.out, ptr %p0, ptr %p
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%stride0> = add ir<%stride>, ir<1>
+; CHECK-NEXT: EMIT ir<%stride0> = add ir<1>, ir<1>
; CHECK-NEXT: EMIT ir<%idx0> = mul ir<%iv>, ir<%stride0>
-; CHECK-NEXT: EMIT ir<%idx1> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx1> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p0>, ir<%idx0>
; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%p1>, ir<%idx1>
; CHECK-NEXT: EMIT-SCALAR ir<%ld0> = load ir<%gep.ld0>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld1> = load ir<%gep.ld1>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld1>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld1> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1074,7 +1126,7 @@ define void @dependent_strides_reverse_order(ptr noalias %p.out, ptr %p0, ptr %p
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1151,7 +1203,8 @@ define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64
; CHECK-NEXT: EMIT-SCALAR ir<%ld1.ext> = sext ir<%ld1> to i64
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1.ext>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -1160,7 +1213,7 @@ define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1169,7 +1222,7 @@ define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1245,7 +1298,8 @@ define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0,
; CHECK-NEXT: EMIT-SCALAR ir<%ld0> = load ir<%gep.ld0>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1.ext>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -1254,7 +1308,7 @@ define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0,
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1263,7 +1317,7 @@ define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0,
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1322,34 +1376,42 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld0> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT ir<%gep.ld1> = getelementptr ir<%gep.ld0>, ir<1>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld0> = load ir<%gep.ld0>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld1> = load ir<%gep.ld1>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld0>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld0> = load vp<[[VP6]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld1>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld1> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld0>, ir<%ld1>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1358,7 +1420,7 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1414,32 +1476,39 @@ define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset)
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%mul> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%mul> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld.base> = getelementptr ir<%p>, ir<%offset>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%gep.ld.base>, ir<%mul>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1448,7 +1517,7 @@ define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1500,32 +1569,39 @@ define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%mul> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%mul> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = add ir<%mul>, ir<%offset>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1534,7 +1610,7 @@ define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1601,7 +1677,8 @@ define void @non_invariant_uniform_base(ptr noalias %p.out, ptr %p, i64 %stride)
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -1610,7 +1687,7 @@ define void @non_invariant_uniform_base(ptr noalias %p.out, ptr %p, i64 %stride)
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1619,7 +1696,7 @@ define void @non_invariant_uniform_base(ptr noalias %p.out, ptr %p, i64 %stride)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1688,7 +1765,8 @@ define void @non_invariant_uniform_stride(ptr noalias %p.out, ptr %p, ptr %p.uni
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -1697,7 +1775,7 @@ define void @non_invariant_uniform_stride(ptr noalias %p.out, ptr %p, ptr %p.uni
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1706,7 +1784,7 @@ define void @non_invariant_uniform_stride(ptr noalias %p.out, ptr %p, ptr %p.uni
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1765,31 +1843,38 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax %n)
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP4]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP6]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1798,7 +1883,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1862,7 +1947,8 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%stride>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -1871,7 +1957,7 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1880,7 +1966,7 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP7]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -1945,7 +2031,8 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -1954,7 +2041,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -1963,7 +2050,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP7]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -2014,32 +2101,39 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: IR %n = mul i64 %m, %stride
-; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax (%stride * %m))
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax %m)
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP4]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP6]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -2048,7 +2142,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -2097,32 +2191,39 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: IR %n = add i64 %stride, 3
-; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax (3 + %stride))
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV 4
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP4]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP6]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -2131,7 +2232,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -2213,35 +2314,43 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: IR %p.out = getelementptr i8, ptr %p2, i64 %out.offset
; CHECK-NEXT: IR %n = add i64 %stride, 3
-; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV (1 smax (3 + %stride))
+; CHECK-NEXT: EMIT vp<[[VP3]]> = EXPAND SCEV 4
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP4]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%gep.ld2> = getelementptr ir<%p2>, ir<%iv>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld2> = load ir<%gep.ld2>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.ld2>, ir<1>
+; CHECK-NEXT: WIDEN ir<%ld2> = load vp<[[VP8]]>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld>, ir<%ld2>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP9:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP9]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<%n>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP6]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP3]]>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -2250,7 +2359,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP11]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -2322,7 +2431,8 @@ define void @actual_stride_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%base>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -2331,7 +2441,7 @@ define void @actual_stride_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -2340,7 +2450,7 @@ define void @actual_stride_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -2391,31 +2501,38 @@ define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<1>, ir<42>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -2424,7 +2541,7 @@ define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -2488,7 +2605,8 @@ define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<1>, ir<%idx>, ir<42>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -2497,7 +2615,7 @@ define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -2506,7 +2624,7 @@ define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -2568,7 +2686,8 @@ define void @nd_array_multiple_idxs(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<1>, ir<%idx>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -2577,7 +2696,7 @@ define void @nd_array_multiple_idxs(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -2586,7 +2705,7 @@ define void @nd_array_multiple_idxs(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -2633,32 +2752,39 @@ define void @sext_stride(ptr noalias %p.out, ptr %p, i32 %stride.i32) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride.i32 == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT-SCALAR ir<%stride> = sext ir<%stride.i32> to i64
+; CHECK-NEXT: EMIT-SCALAR ir<%stride> = sext ir<1> to i64
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -2667,7 +2793,7 @@ define void @sext_stride(ptr noalias %p.out, ptr %p, i32 %stride.i32) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -2716,32 +2842,39 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride.i64 == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
-; CHECK-NEXT: EMIT-SCALAR ir<%stride> = trunc ir<%stride.i64> to i32
+; CHECK-NEXT: EMIT-SCALAR ir<%stride> = trunc ir<1> to i32
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<4>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -2750,7 +2883,7 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -2801,13 +2934,18 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: IR %stride.trunc = trunc i32 %stride to i16
; CHECK-NEXT: IR %stride.ext = sext i32 %stride to i64
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
@@ -2815,23 +2953,26 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; CHECK-NEXT: EMIT-SCALAR ir<%iv.trunc> = trunc ir<%iv> to i16
; CHECK-NEXT: EMIT-SCALAR ir<%iv.ext> = sext ir<%iv> to i64
; CHECK-NEXT: EMIT ir<%idx.trunc> = mul ir<%iv.trunc>, ir<%stride.trunc>
-; CHECK-NEXT: EMIT ir<%idx.ext> = mul ir<%iv.ext>, ir<%stride.ext>
+; CHECK-NEXT: EMIT ir<%idx.ext> = mul ir<%iv.ext>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.trunc> = getelementptr ir<%p0>, ir<%idx.trunc>
; CHECK-NEXT: EMIT ir<%gep.ext> = getelementptr ir<%p0>, ir<%idx.ext>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld.trunc> = load ir<%gep.trunc>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld.ext> = load ir<%gep.ext>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.trunc>, ir<4>
+; CHECK-NEXT: WIDEN ir<%ld.trunc> = load vp<[[VP6]]>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.ext>, ir<4>
+; CHECK-NEXT: WIDEN ir<%ld.ext> = load vp<[[VP7]]>
; CHECK-NEXT: EMIT ir<%val> = add ir<%ld.trunc>, ir<%ld.ext>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -2840,7 +2981,7 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -3007,7 +3148,8 @@ define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
@@ -3016,7 +3158,7 @@ define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -3025,7 +3167,7 @@ define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP6]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -3072,31 +3214,38 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%iv>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<1>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%idx>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<8>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -3105,7 +3254,7 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -3153,33 +3302,41 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%ld>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%ld>
; CHECK-NEXT: EMIT ir<%gep.ptr.st> = getelementptr ir<%p.ptr.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%gep.ld>, ir<%gep.ptr.st>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.ptr.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<%gep.ld>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -3188,7 +3345,7 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -3241,32 +3398,39 @@ define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%idx> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%val> = mul ir<%ld>, ir<%idx>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -3275,7 +3439,7 @@ define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -3325,33 +3489,40 @@ define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride)
; CHECK-NEXT: Live-in ir<128> = original trip-count
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %stride == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP3]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP3:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION nsw ir<0>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%iv.next> = add nsw ir<%iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%iv.times.stride> = mul ir<%iv>, ir<%stride>
+; CHECK-NEXT: EMIT ir<%iv.times.stride> = mul ir<%iv>, ir<1>
; CHECK-NEXT: EMIT ir<%idx> = add ir<%iv.times.stride>, ir<42>
; CHECK-NEXT: EMIT ir<%gep.ld> = getelementptr ir<%p>, ir<%idx>
-; CHECK-NEXT: EMIT-SCALAR ir<%ld> = load ir<%gep.ld>
+; CHECK-NEXT: vp<[[VP6:%[0-9]+]]> = vector-pointer ir<%gep.ld>, ir<8>
+; CHECK-NEXT: WIDEN ir<%ld> = load vp<[[VP6]]>
; CHECK-NEXT: EMIT ir<%val> = mul ir<%ld>, ir<%idx>
; CHECK-NEXT: EMIT ir<%gep.st> = getelementptr ir<%p.out>, ir<%iv>
-; CHECK-NEXT: EMIT store ir<%val>, ir<%gep.st>
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = vector-pointer ir<%gep.st>, ir<1>
+; CHECK-NEXT: WIDEN store vp<[[VP7]]>, ir<%val>
; CHECK-NEXT: EMIT ir<%exitcond> = icmp sge ir<%iv.next>, ir<128>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = exiting-iv-value ir<%iv>
+; CHECK-NEXT: EMIT vp<[[VP9:%[0-9]+]]> = exiting-iv-value ir<%iv>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<128>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -3360,7 +3531,7 @@ define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride)
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP5]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP9]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<header>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<header>:
@@ -3438,33 +3609,39 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<entry>:
; CHECK-NEXT: IR %start.ext = zext i32 %start to i64
-; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEV (zext i32 %start to i64)
-; CHECK-NEXT: EMIT vp<[[VP4]]> = EXPAND SCEV (100 + (-1 * (zext i32 %start to i64))<nsw>)<nsw>
+; CHECK-NEXT: EMIT vp<[[VP3:%[0-9]+]]> = EXPAND SCEV 1
+; CHECK-NEXT: EMIT vp<[[VP4]]> = EXPAND SCEV 99
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = EXPAND SCEVPredicate Equal predicate: %start == 1
+; CHECK-NEXT: Successor(s): scalar.ph, strides.check
+; CHECK-EMPTY:
+; CHECK-NEXT: strides.check:
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP5]]>
; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
; CHECK-EMPTY:
; CHECK-NEXT: vector.ph:
; CHECK-NEXT: Successor(s): vector loop
; CHECK-EMPTY:
; CHECK-NEXT: <x1> vector loop: {
-; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = CANONICAL-IV
+; CHECK-NEXT: vp<[[VP7:%[0-9]+]]> = CANONICAL-IV
; CHECK-EMPTY:
; CHECK-NEXT: vector.body:
-; CHECK-NEXT: ir<%iv.0> = WIDEN-INDUCTION ir<%start.ext>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: ir<%iv.0> = WIDEN-INDUCTION ir<1>, ir<1>, vp<[[VP0]]>
; CHECK-NEXT: ir<%iv.1> = WIDEN-INDUCTION ir<0>, vp<[[VP3]]>, vp<[[VP0]]>
; CHECK-NEXT: EMIT ir<%gep.dst> = getelementptr ir<%dst>, ir<%iv.1>
-; CHECK-NEXT: EMIT store ir<0.000000e+00>, ir<%gep.dst>
+; CHECK-NEXT: vp<[[VP8:%[0-9]+]]> = vector-pointer ir<%gep.dst>, ir<4>
+; CHECK-NEXT: WIDEN store vp<[[VP8]]>, ir<0.000000e+00>
; CHECK-NEXT: EMIT ir<%iv.1.next> = add ir<%iv.1>, vp<[[VP3]]>
; CHECK-NEXT: EMIT ir<%iv.0.next> = add ir<%iv.0>, ir<1>
; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.0.next>, ir<100>
-; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP5]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP7]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
-; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = exiting-iv-value ir<%iv.0>
-; CHECK-NEXT: EMIT vp<[[VP8:%[0-9]+]]> = exiting-iv-value ir<%iv.1>
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = exiting-iv-value ir<%iv.0>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = exiting-iv-value ir<%iv.1>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq vp<[[VP4]]>, vp<[[VP2]]>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
@@ -3473,8 +3650,8 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; CHECK-NEXT: No successors
; CHECK-EMPTY:
; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP7]]>, middle.block ], [ ir<%start.ext>, ir-bb<entry> ]
-; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val>.1 = phi [ vp<[[VP8]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[VP10]]>, middle.block ], [ ir<%start.ext>, ir-bb<entry> ], [ ir<%start.ext>, strides.check ]
+; CHECK-NEXT: EMIT-SCALAR vp<%bc.resume.val>.1 = phi [ vp<[[VP11]]>, middle.block ], [ ir<0>, ir-bb<entry> ], [ ir<0>, strides.check ]
; CHECK-NEXT: Successor(s): ir-bb<loop>
; CHECK-EMPTY:
; CHECK-NEXT: ir-bb<loop>:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
index 4429b0e731616..1c9931e3e881b 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
@@ -1,57 +1,53 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -S \
-; RUN: -enable-mem-access-versioning=false 2>&1 | FileCheck %s --check-prefix COMPARE-NO-MV
+; RUN: -enable-mem-access-versioning=false -enable-vplan-based-stride-mv 2>&1 | FileCheck %s --check-prefix COMPARE-NO-MV
; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -S \
-; RUN: -enable-mem-access-versioning=true 2>&1 | FileCheck %s --check-prefix COMPARE-LAA-MV
+; RUN: -enable-mem-access-versioning=false -enable-vplan-based-stride-mv 2>&1 | FileCheck %s --check-prefix COMPARE-LAA-MV
define void @basic(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-LABEL: define void @basic(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @basic(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -137,7 +133,7 @@ define void @basic_optsize(ptr noalias %p.out, ptr %p, i64 %stride) #0 {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -240,7 +236,7 @@ define void @basic_minsize(ptr noalias %p.out, ptr %p, i64 %stride) #1 {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -318,80 +314,74 @@ define void @byte_gep_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-LABEL: define void @byte_gep_scaled_stride(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = shl <4 x i64> [[BROADCAST_SPLAT]], splat (i64 3)
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = shl i64 [[INDEX]], 3
; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = load <4 x i64>, ptr [[TMP9]], align 8
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[STRIDE_X8:%.*]] = mul i64 [[STRIDE]], 8
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE_X8]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i8, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @byte_gep_scaled_stride(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
-; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = shl <4 x i64> [[BROADCAST_SPLAT]], splat (i64 3)
+; COMPARE-LAA-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH1]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
-; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
-; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
-; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP4]]
-; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = shl i64 [[INDEX]], 3
; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP5]]
-; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
-; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
-; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
-; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = load <4 x i64>, ptr [[TMP9]], align 8
; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-LAA-MV: [[HEADER]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-LAA-MV-NEXT: [[STRIDE_X8:%.*]] = mul i64 [[STRIDE]], 8
+; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE_X8]]
+; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i8, ptr [[P]], i64 [[IDX]]
+; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -455,7 +445,7 @@ define void @byte_gep_under_scaled_stride(ptr noalias %p.out, ptr %p, i64 %strid
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -495,7 +485,7 @@ define void @byte_gep_under_scaled_stride(ptr noalias %p.out, ptr %p, i64 %strid
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -562,7 +552,7 @@ define void @byte_gep_over_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -602,7 +592,7 @@ define void @byte_gep_over_scaled_stride(ptr noalias %p.out, ptr %p, i64 %stride
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -668,7 +658,7 @@ define void @byte_gep_non_power_of_two_scaled_stride(ptr noalias %p.out, ptr %p,
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -708,7 +698,7 @@ define void @byte_gep_non_power_of_two_scaled_stride(ptr noalias %p.out, ptr %p,
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -772,7 +762,7 @@ define void @byte_gep_nonscaled_stride(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -783,18 +773,20 @@ define void @byte_gep_nonscaled_stride(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
-; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-LAA-MV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_SCEVCHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[P]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP3]]
; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP4]], align 8
; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
@@ -804,26 +796,15 @@ define void @byte_gep_nonscaled_stride(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> [[TMP12]], i64 [[TMP9]], i32 1
; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 2
; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 3
-; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX1]]
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP15]], ptr [[TMP16]], align 8
-; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP17]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i8, ptr [[P]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP11:![0-9]+]]
-; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
entry:
@@ -886,7 +867,7 @@ define void @byte_gep_negated_stride(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -960,61 +941,50 @@ define void @shared_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride) {
; COMPARE-NO-MV-LABEL: define void @shared_stride(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP3]]
; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> poison, i64 [[TMP13]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP15]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = insertelement <4 x i64> [[TMP19]], i64 [[TMP16]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP9]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP10]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = load i64, ptr [[TMP11]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = load i64, ptr [[TMP12]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = insertelement <4 x i64> poison, i64 [[TMP21]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = insertelement <4 x i64> [[TMP25]], i64 [[TMP22]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = insertelement <4 x i64> [[TMP26]], i64 [[TMP23]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = insertelement <4 x i64> [[TMP27]], i64 [[TMP24]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = load <4 x i64>, ptr [[TMP12]], align 8
; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = add <4 x i64> [[TMP20]], [[TMP28]]
-; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP29]], ptr [[TMP30]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i64, ptr [[P0]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i64, ptr [[P1]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
+; COMPARE-NO-MV-NEXT: [[LD1:%.*]] = load i64, ptr [[GEP_LD1]], align 8
+; COMPARE-NO-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1]]
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP14:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @shared_stride(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1080,85 +1050,70 @@ define void @independent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stri
; COMPARE-NO-MV-LABEL: define void @independent_strides(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE0:%.*]], i64 [[STRIDE1:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
-; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE0]], 1
+; COMPARE-NO-MV-NEXT: [[IDENT_CHECK1:%.*]] = icmp ne i64 [[STRIDE1]], 1
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = or i1 [[IDENT_CHECK]], [[IDENT_CHECK1]]
+; COMPARE-NO-MV-NEXT: br label %[[STRIDES_CHECK:.*]]
+; COMPARE-NO-MV: [[STRIDES_CHECK]]:
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE0]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE1]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP6]]
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP8]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP10]]
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP12]]
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP14]]
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP16]]
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = load i64, ptr [[TMP3]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP9]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = insertelement <4 x i64> poison, i64 [[TMP18]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = insertelement <4 x i64> [[TMP22]], i64 [[TMP19]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = insertelement <4 x i64> [[TMP23]], i64 [[TMP20]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = insertelement <4 x i64> [[TMP24]], i64 [[TMP21]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = load i64, ptr [[TMP11]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = load i64, ptr [[TMP13]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = load i64, ptr [[TMP15]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = load i64, ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = insertelement <4 x i64> poison, i64 [[TMP26]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = insertelement <4 x i64> [[TMP30]], i64 [[TMP27]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP32:%.*]] = insertelement <4 x i64> [[TMP31]], i64 [[TMP28]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP33:%.*]] = insertelement <4 x i64> [[TMP32]], i64 [[TMP29]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP34:%.*]] = add <4 x i64> [[TMP25]], [[TMP33]]
-; COMPARE-NO-MV-NEXT: [[TMP35:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP34]], ptr [[TMP35]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P0]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P1]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = add <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP3]], align 8
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-NO-MV-NEXT: [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP36]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX0:%.*]] = mul i64 [[IV]], [[STRIDE0]]
+; COMPARE-NO-MV-NEXT: [[IDX1:%.*]] = mul i64 [[IV]], [[STRIDE1]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i64, ptr [[P0]], i64 [[IDX0]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i64, ptr [[P1]], i64 [[IDX1]]
+; COMPARE-NO-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
+; COMPARE-NO-MV-NEXT: [[LD1:%.*]] = load i64, ptr [[GEP_LD1]], align 8
+; COMPARE-NO-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1]]
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP16:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @independent_strides(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE0:%.*]], i64 [[STRIDE1:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
-; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE0]], 1
; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK1:%.*]] = icmp ne i64 [[STRIDE1]], 1
-; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = or i1 [[IDENT_CHECK]], [[IDENT_CHECK1]]
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP0]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = or i1 [[IDENT_CHECK]], [[IDENT_CHECK1]]
+; COMPARE-LAA-MV-NEXT: br label %[[STRIDES_CHECK:.*]]
+; COMPARE-LAA-MV: [[STRIDES_CHECK]]:
+; COMPARE-LAA-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P0]], i64 [[INDEX]]
-; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[P1]], i64 [[INDEX]]
-; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
-; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = add <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP3]], ptr [[TMP4]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P0]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P1]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = add <4 x i64> [[WIDE_LOAD]], [[WIDE_LOAD2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP3]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -1211,41 +1166,26 @@ define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride
; COMPARE-NO-MV-LABEL: define void @dependent_strides(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], splat (i64 1)
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = shl <4 x i64> [[VEC_IND]], splat (i64 1)
; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP5]]
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP6]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP7]]
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP6]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP8]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[TMP6]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP9]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP6]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP10]]
-; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = load i64, ptr [[TMP11]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = load i64, ptr [[TMP12]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP13]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP14]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = insertelement <4 x i64> poison, i64 [[TMP19]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = insertelement <4 x i64> [[TMP23]], i64 [[TMP20]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = insertelement <4 x i64> [[TMP24]], i64 [[TMP21]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = insertelement <4 x i64> [[TMP25]], i64 [[TMP22]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = load <4 x i64>, ptr [[TMP14]], align 8
; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = load i64, ptr [[TMP15]], align 8
; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = load i64, ptr [[TMP16]], align 8
; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = load i64, ptr [[TMP17]], align 8
@@ -1255,23 +1195,40 @@ define void @dependent_strides(ptr noalias %p.out, ptr %p0, ptr %p1, i64 %stride
; COMPARE-NO-MV-NEXT: [[TMP33:%.*]] = insertelement <4 x i64> [[TMP32]], i64 [[TMP29]], i32 2
; COMPARE-NO-MV-NEXT: [[TMP34:%.*]] = insertelement <4 x i64> [[TMP33]], i64 [[TMP30]], i32 3
; COMPARE-NO-MV-NEXT: [[TMP35:%.*]] = add <4 x i64> [[TMP26]], [[TMP34]]
-; COMPARE-NO-MV-NEXT: [[TMP36:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP36:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP5]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP35]], ptr [[TMP36]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP5]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP37]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP37]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[STRIDE1:%.*]] = add i64 [[STRIDE]], 1
+; COMPARE-NO-MV-NEXT: [[IDX0:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[IDX1:%.*]] = mul i64 [[IV]], [[STRIDE1]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i64, ptr [[P0]], i64 [[IDX0]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i64, ptr [[P1]], i64 [[IDX1]]
+; COMPARE-NO-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
+; COMPARE-NO-MV-NEXT: [[LD1:%.*]] = load i64, ptr [[GEP_LD1]], align 8
+; COMPARE-NO-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1]]
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP18:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @dependent_strides(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1360,67 +1317,69 @@ define void @dependent_strides_reverse_order(ptr noalias %p.out, ptr %p0, ptr %p
; COMPARE-NO-MV-LABEL: define void @dependent_strides_reverse_order(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], splat (i64 1)
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = shl <4 x i64> [[VEC_IND]], splat (i64 1)
+; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP6]], i64 0
+; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP7]]
+; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP6]], i64 1
; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP6]], i64 2
; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP6]], i64 3
; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = getelementptr i64, ptr [[P0]], i64 [[TMP5]]
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP6]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP7]]
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[TMP6]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP8]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[TMP6]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP9]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = extractelement <4 x i64> [[TMP6]], i64 3
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P1]], i64 [[TMP10]]
-; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = load i64, ptr [[TMP11]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = load i64, ptr [[TMP12]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP13]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP14]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = insertelement <4 x i64> poison, i64 [[TMP19]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = insertelement <4 x i64> [[TMP23]], i64 [[TMP20]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = insertelement <4 x i64> [[TMP24]], i64 [[TMP21]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = insertelement <4 x i64> [[TMP25]], i64 [[TMP22]], i32 3
; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = load i64, ptr [[TMP15]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = load i64, ptr [[TMP16]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = load i64, ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = load i64, ptr [[TMP18]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = load i64, ptr [[TMP12]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = load i64, ptr [[TMP13]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = load i64, ptr [[TMP14]], align 8
; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = insertelement <4 x i64> poison, i64 [[TMP27]], i32 0
; COMPARE-NO-MV-NEXT: [[TMP32:%.*]] = insertelement <4 x i64> [[TMP31]], i64 [[TMP28]], i32 1
; COMPARE-NO-MV-NEXT: [[TMP33:%.*]] = insertelement <4 x i64> [[TMP32]], i64 [[TMP29]], i32 2
; COMPARE-NO-MV-NEXT: [[TMP34:%.*]] = insertelement <4 x i64> [[TMP33]], i64 [[TMP30]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP35:%.*]] = add <4 x i64> [[TMP26]], [[TMP34]]
-; COMPARE-NO-MV-NEXT: [[TMP36:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP18]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP35:%.*]] = add <4 x i64> [[TMP34]], [[WIDE_LOAD]]
+; COMPARE-NO-MV-NEXT: [[TMP36:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP10]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP35]], ptr [[TMP36]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP10]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP37]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP37]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[STRIDE0:%.*]] = add i64 [[STRIDE]], 1
+; COMPARE-NO-MV-NEXT: [[IDX0:%.*]] = mul i64 [[IV]], [[STRIDE0]]
+; COMPARE-NO-MV-NEXT: [[IDX1:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i64, ptr [[P0]], i64 [[IDX0]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i64, ptr [[P1]], i64 [[IDX1]]
+; COMPARE-NO-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
+; COMPARE-NO-MV-NEXT: [[LD1:%.*]] = load i64, ptr [[GEP_LD1]], align 8
+; COMPARE-NO-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1]]
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP20:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @dependent_strides_reverse_order(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1552,7 +1511,7 @@ define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -1563,18 +1522,20 @@ define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
-; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-LAA-MV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_SCEVCHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[P0]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[P0]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P0]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P0]], i64 [[TMP3]]
; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP4]], align 8
; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
@@ -1598,30 +1559,15 @@ define void @byte_dependent_byte_geps(ptr noalias %p.out, ptr %p0, ptr %p1, i64
; COMPARE-LAA-MV-NEXT: [[TMP27:%.*]] = insertelement <4 x i32> [[TMP26]], i32 [[TMP23]], i32 3
; COMPARE-LAA-MV-NEXT: [[TMP28:%.*]] = sext <4 x i32> [[TMP27]] to <4 x i64>
; COMPARE-LAA-MV-NEXT: [[TMP29:%.*]] = add <4 x i64> [[TMP15]], [[TMP28]]
-; COMPARE-LAA-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX1]]
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP29]], ptr [[TMP30]], align 8
-; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i8, ptr [[P0]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
-; COMPARE-LAA-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i8, ptr [[P1]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD1:%.*]] = load i32, ptr [[GEP_LD1]], align 8
-; COMPARE-LAA-MV-NEXT: [[LD1_EXT:%.*]] = sext i32 [[LD1]] to i64
-; COMPARE-LAA-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1_EXT]]
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP22:![0-9]+]]
-; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
entry:
@@ -1701,7 +1647,7 @@ define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0,
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -1712,18 +1658,20 @@ define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0,
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
-; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-LAA-MV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_SCEVCHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[P1]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP3]]
; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP4]], align 8
; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 8
@@ -1747,30 +1695,15 @@ define void @byte_dependent_byte_geps_reverse_order(ptr noalias %p.out, ptr %p0,
; COMPARE-LAA-MV-NEXT: [[TMP27:%.*]] = insertelement <4 x i64> [[TMP26]], i64 [[TMP23]], i32 2
; COMPARE-LAA-MV-NEXT: [[TMP28:%.*]] = insertelement <4 x i64> [[TMP27]], i64 [[TMP24]], i32 3
; COMPARE-LAA-MV-NEXT: [[TMP29:%.*]] = add <4 x i64> [[TMP28]], [[TMP16]]
-; COMPARE-LAA-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX1]]
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP29]], ptr [[TMP30]], align 8
-; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP31]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i8, ptr [[P1]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD1:%.*]] = load i32, ptr [[GEP_LD1]], align 8
-; COMPARE-LAA-MV-NEXT: [[LD1_EXT:%.*]] = sext i32 [[LD1]] to i64
-; COMPARE-LAA-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i8, ptr [[P0]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
-; COMPARE-LAA-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1_EXT]]
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP24:![0-9]+]]
-; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
entry:
@@ -1808,61 +1741,50 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-LABEL: define void @strided_interleave(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP5]], i61 1
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[TMP6]], i61 1
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[TMP7]], i61 1
; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[TMP8]], i61 1
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> poison, i64 [[TMP13]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP15]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = insertelement <4 x i64> [[TMP19]], i64 [[TMP16]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = load i64, ptr [[TMP9]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP10]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = load i64, ptr [[TMP11]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = load i64, ptr [[TMP12]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = insertelement <4 x i64> poison, i64 [[TMP21]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = insertelement <4 x i64> [[TMP25]], i64 [[TMP22]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = insertelement <4 x i64> [[TMP26]], i64 [[TMP23]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = insertelement <4 x i64> [[TMP27]], i64 [[TMP24]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = load <4 x i64>, ptr [[TMP12]], align 8
; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = add <4 x i64> [[TMP20]], [[TMP28]]
-; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP29]], ptr [[TMP30]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD0:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD1:%.*]] = getelementptr i64, ptr [[GEP_LD0]], i61 1
+; COMPARE-NO-MV-NEXT: [[LD0:%.*]] = load i64, ptr [[GEP_LD0]], align 8
+; COMPARE-NO-MV-NEXT: [[LD1:%.*]] = load i64, ptr [[GEP_LD1]], align 8
+; COMPARE-NO-MV-NEXT: [[VAL:%.*]] = add i64 [[LD0]], [[LD1]]
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP24:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @strided_interleave(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -1877,7 +1799,7 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP2]], ptr [[TMP3]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -1894,7 +1816,7 @@ define void @strided_interleave(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP26:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP24:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -1931,49 +1853,47 @@ define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset)
; COMPARE-NO-MV-LABEL: define void @in_loop_base(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OFFSET:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET]]
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP0]], i64 [[TMP5]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = load <4 x i64>, ptr [[TMP9]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP5]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP5]], 4
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[MUL:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = add i64 [[MUL]], [[OFFSET]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD_BASE:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[GEP_LD_BASE]], i64 [[MUL]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP26:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @in_loop_base(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OFFSET:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET]]
@@ -1986,7 +1906,7 @@ define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset)
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP2]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -2002,7 +1922,7 @@ define void @in_loop_base(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset)
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP28:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP26:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2036,51 +1956,46 @@ define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset
; COMPARE-NO-MV-LABEL: define void @base_not_in_ir(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OFFSET:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[OFFSET]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = add <4 x i64> [[TMP0]], [[BROADCAST_SPLAT2]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], [[OFFSET]]
; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP5]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = load <4 x i64>, ptr [[TMP9]], align 8
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[MUL:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = add i64 [[MUL]], [[OFFSET]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP28:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @base_not_in_ir(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OFFSET:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -2093,7 +2008,7 @@ define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP2]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -2108,7 +2023,7 @@ define void @base_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride, i64 %offset
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP30:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP28:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2173,7 +2088,7 @@ define void @non_invariant_uniform_base(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -2214,7 +2129,7 @@ define void @non_invariant_uniform_base(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -2282,7 +2197,7 @@ define void @non_invariant_uniform_stride(ptr noalias %p.out, ptr %p, ptr %p.uni
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -2324,7 +2239,7 @@ define void @non_invariant_uniform_stride(ptr noalias %p.out, ptr %p, ptr %p.uni
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -2365,46 +2280,30 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-NO-MV-LABEL: define void @non_constant_btc(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_PH]] ]
; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-NO-MV: [[HEADER]]:
; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
@@ -2415,18 +2314,18 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP23:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP32:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @non_constant_btc(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
; COMPARE-LAA-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-LAA-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
@@ -2440,7 +2339,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -2456,7 +2355,7 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP34:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP32:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2522,7 +2421,7 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -2538,16 +2437,56 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP25:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP34:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @stride_as_btc(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[STRIDE]], i64 1)
+; COMPARE-LAA-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-LAA-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-LAA-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[HEADER]], !llvm.loop [[LOOP33:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-LAA-MV: [[HEADER1]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -2555,7 +2494,7 @@ define void @stride_as_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[EXIT]], !llvm.loop [[LOOP34:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2622,7 +2561,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -2638,7 +2577,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP27:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP36:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -2646,9 +2585,49 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 1
+; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-LAA-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; COMPARE-LAA-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[HEADER]], !llvm.loop [[LOOP35:![0-9]+]]
+; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
+; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-LAA-MV-NEXT: br label %[[HEADER1:.*]]
+; COMPARE-LAA-MV: [[HEADER1]]:
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER1]] ]
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -2656,7 +2635,7 @@ define void @stride_dependent_btc(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER1]], label %[[EXIT]], !llvm.loop [[LOOP36:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2691,46 +2670,30 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
-; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[M]], i64 1)
; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_PH]] ]
; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-NO-MV: [[HEADER]]:
; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
@@ -2741,7 +2704,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP29:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP38:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -2749,12 +2712,11 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[M]], i64 1)
; COMPARE-LAA-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX1]], 4
; COMPARE-LAA-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX1]], 4
@@ -2768,7 +2730,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX1]], [[N_VEC]]
; COMPARE-LAA-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -2784,7 +2746,7 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP36:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP38:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2815,51 +2777,24 @@ exit:
define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-LABEL: define void @stride_dependent_btc_non_preventive(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
-; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-NO-MV-NEXT: br label %[[STRIDES_CHECK:.*]]
+; COMPARE-NO-MV: [[STRIDES_CHECK]]:
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[P]], align 8
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[P_OUT]], align 8
+; COMPARE-NO-MV-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-NO-MV: [[HEADER]]:
-; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -2867,7 +2802,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP31:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP39:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -2875,10 +2810,9 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -2899,7 +2833,7 @@ define void @stride_dependent_btc_non_preventive(ptr noalias %p.out, ptr %p, i64
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP37:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP39:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -2955,31 +2889,12 @@ define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset
;
; COMPARE-LAA-MV-LABEL: define void @stride_btc_memdep_triple_check(
; COMPARE-LAA-MV-SAME: ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
-; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P]], i64 [[OUT_OFFSET]]
; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
-; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
-; COMPARE-LAA-MV: [[VECTOR_MEMCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = sub i64 [[OUT_OFFSET]], 1
-; COMPARE-LAA-MV-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 31
-; COMPARE-LAA-MV-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
-; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[P]], align 8
-; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[P_OUT]], align 8
-; COMPARE-LAA-MV-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-LAA-MV: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
@@ -2987,7 +2902,7 @@ define void @stride_btc_memdep_triple_check(ptr %p, i64 %stride, i64 %out.offset
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP38:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3013,62 +2928,35 @@ header:
exit:
ret void
-}
-
-; Same as above but memdep check doesn't depend on stride
-define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2, i64 %stride, i64 %out.offset) {
-; COMPARE-NO-MV-LABEL: define void @stride_btc_independent_memdep_triple_check(
-; COMPARE-NO-MV-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
-; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
-; COMPARE-NO-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
-; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
-; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
-; COMPARE-NO-MV: [[VECTOR_MEMCHECK]]:
-; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = sub i64 [[OUT_OFFSET]], 1
-; COMPARE-NO-MV-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP21]], 31
-; COMPARE-NO-MV-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
-; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
-; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
-; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P2]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP17]], align 8
+}
+
+; Same as above but memdep check doesn't depend on stride
+define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2, i64 %stride, i64 %out.offset) {
+; COMPARE-NO-MV-LABEL: define void @stride_btc_independent_memdep_triple_check(
+; COMPARE-NO-MV-SAME: ptr [[P:%.*]], ptr noalias [[P2:%.*]], i64 [[STRIDE:%.*]], i64 [[OUT_OFFSET:%.*]]) {
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
+; COMPARE-NO-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
+; COMPARE-NO-MV-NEXT: br label %[[STRIDES_CHECK:.*]]
+; COMPARE-NO-MV: [[STRIDES_CHECK]]:
+; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; COMPARE-NO-MV: [[VECTOR_MEMCHECK]]:
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = sub i64 [[OUT_OFFSET]], 1
+; COMPARE-NO-MV-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 31
+; COMPARE-NO-MV-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
+; COMPARE-NO-MV: [[VECTOR_BODY]]:
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = load <4 x i64>, ptr [[P]], align 8
+; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[P2]], align 8
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = add <4 x i64> [[TMP16]], [[WIDE_LOAD]]
-; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
-; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP18]], ptr [[TMP19]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP18]], ptr [[P_OUT]], align 8
+; COMPARE-NO-MV-NEXT: br label %[[MIDDLE_BLOCK:.*]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[STRIDES_CHECK]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-NO-MV: [[HEADER]]:
; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
@@ -3082,7 +2970,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-NO-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP33:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP40:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -3091,10 +2979,9 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: [[P_OUT:%.*]] = getelementptr i8, ptr [[P2]], i64 [[OUT_OFFSET]]
; COMPARE-LAA-MV-NEXT: [[N:%.*]] = add i64 [[STRIDE]], 3
-; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_MEMCHECK]]:
; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = sub i64 [[OUT_OFFSET]], 1
@@ -3125,7 +3012,7 @@ define void @stride_btc_independent_memdep_triple_check(ptr %p, ptr noalias %p2,
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP39:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP40:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3196,7 +3083,7 @@ define void @actual_stride_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -3239,7 +3126,7 @@ define void @actual_stride_not_in_ir(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -3275,48 +3162,44 @@ define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-LABEL: define void @nd_array_last_idx(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 42, i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 42, i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 42, i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 42, i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP16]], ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 42, i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP43:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @nd_array_last_idx(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -3328,7 +3211,7 @@ define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -3342,7 +3225,7 @@ define void @nd_array_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP42:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP43:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -3405,7 +3288,7 @@ define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -3416,18 +3299,20 @@ define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
-; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-LAA-MV-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_SCEVCHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[INDEX]], i64 42
+; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[TMP1]], i64 42
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[TMP2]], i64 42
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[TMP3]], i64 42
; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP4]], align 8
; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
@@ -3437,26 +3322,15 @@ define void @nd_array_non_last_idx(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> [[TMP12]], i64 [[TMP9]], i32 1
; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 2
; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 3
-; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX1]]
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[TMP15]], ptr [[TMP16]], align 8
-; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], 4
+; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP17]], label %[[SCALAR_PH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr [256 x [256 x i64]], ptr [[P]], i64 1, i64 [[IDX]], i64 42
-; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP44:![0-9]+]]
-; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
entry:
@@ -3516,7 +3390,7 @@ define void @nd_array_multiple_idxs(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -3588,49 +3462,45 @@ define void @sext_stride(ptr noalias %p.out, ptr %p, i32 %stride.i32) {
; COMPARE-NO-MV-LABEL: define void @sext_stride(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i32 [[STRIDE_I32:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i32 [[STRIDE_I32]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[STRIDE_I32]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = sext <4 x i32> [[BROADCAST_SPLAT]] to <4 x i64>
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = mul <4 x i64> [[VEC_IND]], [[TMP0]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP5]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = load <4 x i64>, ptr [[TMP9]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP5]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP5]], 4
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[STRIDE:%.*]] = sext i32 [[STRIDE_I32]] to i64
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP47:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @sext_stride(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i32 [[STRIDE_I32:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[STRIDE_I32]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[STRIDE_I32]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -3689,8 +3559,11 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; COMPARE-NO-MV-LABEL: define void @trunc_stride(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE_I64:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE_I64]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-NO-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK1:.*]]
+; COMPARE-NO-MV: [[VECTOR_SCEVCHECK1]]:
; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = trunc i64 [[STRIDE_I64]] to i32
; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = sub i32 0, [[TMP0]]
; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = icmp slt i32 [[TMP0]], 0
@@ -3703,44 +3576,25 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = icmp sgt i32 [[TMP4]], 0
; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = select i1 [[TMP2]], i1 [[TMP6]], i1 [[TMP5]]
; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP8]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP8]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE_I64]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = trunc <4 x i64> [[BROADCAST_SPLAT]] to <4 x i32>
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = mul <4 x i32> [[VEC_IND]], [[TMP9]]
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = extractelement <4 x i32> [[TMP10]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP11]]
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = extractelement <4 x i32> [[TMP10]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP12]]
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[TMP10]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP13]]
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = extractelement <4 x i32> [[TMP10]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i32, ptr [[P]], i32 [[TMP14]]
-; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = load i32, ptr [[TMP15]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = load i32, ptr [[TMP16]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP17]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP18]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> poison, i32 [[TMP19]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TMP20]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP24]], i32 [[TMP21]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> [[TMP25]], i32 [[TMP22]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = load <4 x i32>, ptr [[TMP18]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[TMP14]]
; COMPARE-NO-MV-NEXT: store <4 x i32> [[TMP26]], ptr [[TMP27]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 4)
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[TMP14]], 4
; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_SCEVCHECK1]] ]
; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-NO-MV: [[HEADER]]:
-; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-NEXT: [[STRIDE:%.*]] = trunc i64 [[STRIDE_I64]] to i32
; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1
; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i32 [[IV]], [[STRIDE]]
@@ -3749,21 +3603,35 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[IV]]
; COMPARE-NO-MV-NEXT: store i32 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[IV_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP40:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP49:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @trunc_stride(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE_I64:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE_I64]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE_I64]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = trunc i64 [[STRIDE_I64]] to i32
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = sub i32 0, [[TMP9]]
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = icmp slt i32 [[TMP9]], 0
+; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 [[TMP9]]
+; COMPARE-LAA-MV-NEXT: [[MUL:%.*]] = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 [[TMP3]], i32 127)
+; COMPARE-LAA-MV-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i32, i1 } [[MUL]], 0
+; COMPARE-LAA-MV-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i32, i1 } [[MUL]], 1
+; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = sub i32 0, [[MUL_RESULT]]
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = icmp slt i32 [[MUL_RESULT]], 0
+; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = icmp sgt i32 [[TMP4]], 0
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = select i1 [[TMP11]], i1 [[TMP6]], i1 [[TMP5]]
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = or i1 [[TMP7]], [[MUL_OVERFLOW]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP8]], label %[[SCALAR_PH]], label %[[VECTOR_PH1:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH1]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[P]], i32 [[INDEX]]
; COMPARE-LAA-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 8
; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[INDEX]]
@@ -3774,9 +3642,10 @@ define void @trunc_stride(ptr noalias %p.out, ptr %p, i64 %stride.i64) {
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_PH]] ]
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-NEXT: [[STRIDE:%.*]] = trunc i64 [[STRIDE_I64]] to i32
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i32 [[IV]], [[STRIDE]]
@@ -3819,8 +3688,11 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-NO-MV-NEXT: [[STRIDE_TRUNC:%.*]] = trunc i32 [[STRIDE]] to i16
; COMPARE-NO-MV-NEXT: [[STRIDE_EXT:%.*]] = sext i32 [[STRIDE]] to i64
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i32 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-NO-MV: [[VECTOR_SCEVCHECK]]:
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK1:.*]]
+; COMPARE-NO-MV: [[VECTOR_SCEVCHECK1]]:
; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = sub i16 0, [[STRIDE_TRUNC]]
; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = icmp slt i16 [[STRIDE_TRUNC]], 0
; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i16 [[TMP0]], i16 [[STRIDE_TRUNC]]
@@ -3832,66 +3704,31 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = icmp sgt i16 [[TMP3]], 0
; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = select i1 [[TMP1]], i1 [[TMP5]], i1 [[TMP4]]
; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = or i1 [[TMP6]], [[MUL_OVERFLOW]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP7]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP7]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[STRIDE_TRUNC]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE_EXT]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND3:%.*]] = phi <4 x i16> [ <i16 0, i16 1, i16 2, i16 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT4:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = sext <4 x i32> [[VEC_IND]] to <4 x i64>
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = mul <4 x i16> [[VEC_IND3]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = mul <4 x i64> [[TMP8]], [[BROADCAST_SPLAT2]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = extractelement <4 x i16> [[TMP9]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr [[P0]], i16 [[TMP10]]
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = extractelement <4 x i16> [[TMP9]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[P0]], i16 [[TMP11]]
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = extractelement <4 x i16> [[TMP9]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP21:%.*]] = getelementptr i32, ptr [[P0]], i16 [[TMP12]]
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = extractelement <4 x i16> [[TMP9]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = trunc i32 [[INDEX]] to i16
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = sext i32 [[INDEX]] to i64
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = mul i16 [[TMP8]], [[STRIDE_TRUNC]]
; COMPARE-NO-MV-NEXT: [[TMP22:%.*]] = getelementptr i32, ptr [[P0]], i16 [[TMP13]]
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = extractelement <4 x i64> [[TMP14]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP23:%.*]] = getelementptr i32, ptr [[P0]], i64 [[TMP15]]
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[TMP14]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP24:%.*]] = getelementptr i32, ptr [[P0]], i64 [[TMP16]]
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = extractelement <4 x i64> [[TMP14]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP25:%.*]] = getelementptr i32, ptr [[P0]], i64 [[TMP17]]
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = extractelement <4 x i64> [[TMP14]], i64 3
; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = getelementptr i32, ptr [[P0]], i64 [[TMP18]]
-; COMPARE-NO-MV-NEXT: [[TMP27:%.*]] = load i32, ptr [[TMP19]], align 4
-; COMPARE-NO-MV-NEXT: [[TMP28:%.*]] = load i32, ptr [[TMP20]], align 4
-; COMPARE-NO-MV-NEXT: [[TMP29:%.*]] = load i32, ptr [[TMP21]], align 4
-; COMPARE-NO-MV-NEXT: [[TMP30:%.*]] = load i32, ptr [[TMP22]], align 4
-; COMPARE-NO-MV-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> poison, i32 [[TMP27]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP32:%.*]] = insertelement <4 x i32> [[TMP31]], i32 [[TMP28]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP33:%.*]] = insertelement <4 x i32> [[TMP32]], i32 [[TMP29]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP34:%.*]] = insertelement <4 x i32> [[TMP33]], i32 [[TMP30]], i32 3
-; COMPARE-NO-MV-NEXT: [[TMP35:%.*]] = load i32, ptr [[TMP23]], align 4
-; COMPARE-NO-MV-NEXT: [[TMP36:%.*]] = load i32, ptr [[TMP24]], align 4
-; COMPARE-NO-MV-NEXT: [[TMP37:%.*]] = load i32, ptr [[TMP25]], align 4
-; COMPARE-NO-MV-NEXT: [[TMP38:%.*]] = load i32, ptr [[TMP26]], align 4
-; COMPARE-NO-MV-NEXT: [[TMP39:%.*]] = insertelement <4 x i32> poison, i32 [[TMP35]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP40:%.*]] = insertelement <4 x i32> [[TMP39]], i32 [[TMP36]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP41:%.*]] = insertelement <4 x i32> [[TMP40]], i32 [[TMP37]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> [[TMP41]], i32 [[TMP38]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP34:%.*]] = load <4 x i32>, ptr [[TMP22]], align 4
+; COMPARE-NO-MV-NEXT: [[TMP42:%.*]] = load <4 x i32>, ptr [[TMP26]], align 4
; COMPARE-NO-MV-NEXT: [[TMP43:%.*]] = add <4 x i32> [[TMP34]], [[TMP42]]
; COMPARE-NO-MV-NEXT: [[TMP44:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[INDEX]]
; COMPARE-NO-MV-NEXT: store <4 x i32> [[TMP43]], ptr [[TMP44]], align 8
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i32> [[VEC_IND]], splat (i32 4)
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT4]] = add <4 x i16> [[VEC_IND3]], splat (i16 4)
; COMPARE-NO-MV-NEXT: [[TMP45:%.*]] = icmp eq i32 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP45]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP45]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_SCEVCHECK1]] ]
; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-NO-MV: [[HEADER]]:
-; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1
; COMPARE-NO-MV-NEXT: [[IV_TRUNC:%.*]] = trunc i32 [[IV]] to i16
; COMPARE-NO-MV-NEXT: [[IV_EXT:%.*]] = sext i32 [[IV]] to i64
@@ -3905,7 +3742,7 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i32, ptr [[P_OUT]], i32 [[IV]]
; COMPARE-NO-MV-NEXT: store i32 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[IV_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP42:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP51:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -3914,14 +3751,27 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: [[STRIDE_TRUNC:%.*]] = trunc i32 [[STRIDE]] to i16
; COMPARE-LAA-MV-NEXT: [[STRIDE_EXT:%.*]] = sext i32 [[STRIDE]] to i64
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = sub i16 0, [[STRIDE_TRUNC]]
+; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = icmp slt i16 [[STRIDE_TRUNC]], 0
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i16 [[TMP8]], i16 [[STRIDE_TRUNC]]
+; COMPARE-LAA-MV-NEXT: [[MUL:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[TMP10]], i16 127)
+; COMPARE-LAA-MV-NEXT: [[MUL_RESULT:%.*]] = extractvalue { i16, i1 } [[MUL]], 0
+; COMPARE-LAA-MV-NEXT: [[MUL_OVERFLOW:%.*]] = extractvalue { i16, i1 } [[MUL]], 1
+; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = sub i16 0, [[MUL_RESULT]]
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = icmp slt i16 [[MUL_RESULT]], 0
+; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = icmp sgt i16 [[TMP11]], 0
+; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = select i1 [[TMP9]], i1 [[TMP13]], i1 [[TMP12]]
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = or i1 [[TMP14]], [[MUL_OVERFLOW]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP15]], label %[[SCALAR_PH]], label %[[VECTOR_PH1:.*]]
+; COMPARE-LAA-MV: [[VECTOR_PH1]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = trunc i32 [[INDEX]] to i16
; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = sext i32 [[INDEX]] to i64
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = mul i16 [[TMP0]], [[STRIDE_TRUNC]]
@@ -3938,9 +3788,10 @@ define void @trunc_ext_stride(ptr noalias %p.out, ptr %p0, ptr %p1, i32 %stride)
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
+; COMPARE-LAA-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[VECTOR_SCEVCHECK]] ], [ 0, %[[VECTOR_PH]] ]
; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1
; COMPARE-LAA-MV-NEXT: [[IV_TRUNC:%.*]] = trunc i32 [[IV]] to i16
; COMPARE-LAA-MV-NEXT: [[IV_EXT:%.*]] = sext i32 [[IV]] to i64
@@ -4055,7 +3906,7 @@ define void @basic_masked(ptr noalias %p.out, ptr %p, i64 %stride, i64 %x) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP6]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -4066,79 +3917,66 @@ define void @basic_masked(ptr noalias %p.out, ptr %p, i64 %stride, i64 %x) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
-; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[X]], i64 0
; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
+; COMPARE-LAA-MV-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-LAA-MV: [[VECTOR_BODY]]:
-; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
-; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ]
+; COMPARE-LAA-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_SCEVCHECK]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
+; COMPARE-LAA-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_SCEVCHECK]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ]
; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = icmp sge <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-LAA-MV-NEXT: [[TMP22:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
; COMPARE-LAA-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i64 0
; COMPARE-LAA-MV-NEXT: br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; COMPARE-LAA-MV: [[PRED_STORE_IF]]:
+; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP22]], i64 0
; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP2]]
+; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-LAA-MV-NEXT: store i64 [[TMP4]], ptr [[TMP5]], align 8
; COMPARE-LAA-MV-NEXT: br label %[[PRED_STORE_CONTINUE]]
; COMPARE-LAA-MV: [[PRED_STORE_CONTINUE]]:
; COMPARE-LAA-MV-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP0]], i64 1
; COMPARE-LAA-MV-NEXT: br i1 [[TMP6]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
; COMPARE-LAA-MV: [[PRED_STORE_IF1]]:
-; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = add i64 [[TMP2]], 1
+; COMPARE-LAA-MV-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP22]], i64 1
; COMPARE-LAA-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP7]]
; COMPARE-LAA-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP7]]
+; COMPARE-LAA-MV-NEXT: [[TMP24:%.*]] = add i64 [[INDEX]], 1
+; COMPARE-LAA-MV-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP24]]
; COMPARE-LAA-MV-NEXT: store i64 [[TMP9]], ptr [[TMP10]], align 8
; COMPARE-LAA-MV-NEXT: br label %[[PRED_STORE_CONTINUE2]]
; COMPARE-LAA-MV: [[PRED_STORE_CONTINUE2]]:
; COMPARE-LAA-MV-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP0]], i64 2
; COMPARE-LAA-MV-NEXT: br i1 [[TMP11]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
; COMPARE-LAA-MV: [[PRED_STORE_IF3]]:
-; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = add i64 [[TMP2]], 2
+; COMPARE-LAA-MV-NEXT: [[TMP12:%.*]] = extractelement <4 x i64> [[TMP22]], i64 2
; COMPARE-LAA-MV-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP12]]
; COMPARE-LAA-MV-NEXT: [[TMP14:%.*]] = load i64, ptr [[TMP13]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP12]]
+; COMPARE-LAA-MV-NEXT: [[TMP25:%.*]] = add i64 [[INDEX]], 2
+; COMPARE-LAA-MV-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP25]]
; COMPARE-LAA-MV-NEXT: store i64 [[TMP14]], ptr [[TMP15]], align 8
; COMPARE-LAA-MV-NEXT: br label %[[PRED_STORE_CONTINUE4]]
; COMPARE-LAA-MV: [[PRED_STORE_CONTINUE4]]:
; COMPARE-LAA-MV-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP0]], i64 3
; COMPARE-LAA-MV-NEXT: br i1 [[TMP16]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
; COMPARE-LAA-MV: [[PRED_STORE_IF5]]:
-; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = add i64 [[TMP2]], 3
+; COMPARE-LAA-MV-NEXT: [[TMP17:%.*]] = extractelement <4 x i64> [[TMP22]], i64 3
; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP17]]
; COMPARE-LAA-MV-NEXT: [[TMP19:%.*]] = load i64, ptr [[TMP18]], align 8
-; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP17]]
+; COMPARE-LAA-MV-NEXT: [[TMP23:%.*]] = add i64 [[INDEX]], 3
+; COMPARE-LAA-MV-NEXT: [[TMP20:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP23]]
; COMPARE-LAA-MV-NEXT: store i64 [[TMP19]], ptr [[TMP20]], align 8
; COMPARE-LAA-MV-NEXT: br label %[[PRED_STORE_CONTINUE6]]
; COMPARE-LAA-MV: [[PRED_STORE_CONTINUE6]]:
-; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 4
+; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]]
-; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
-; COMPARE-LAA-MV: [[SCALAR_PH]]:
-; COMPARE-LAA-MV-NEXT: br label %[[HEADER:.*]]
-; COMPARE-LAA-MV: [[HEADER]]:
-; COMPARE-LAA-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
-; COMPARE-LAA-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; COMPARE-LAA-MV-NEXT: [[C:%.*]] = icmp sge i64 [[IV]], [[X]]
-; COMPARE-LAA-MV-NEXT: br i1 [[C]], label %[[IF:.*]], label %[[LATCH]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP21]], label %[[IF:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]]
; COMPARE-LAA-MV: [[IF]]:
-; COMPARE-LAA-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
-; COMPARE-LAA-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
-; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
-; COMPARE-LAA-MV-NEXT: br label %[[LATCH]]
+; COMPARE-LAA-MV-NEXT: br label %[[LATCH:.*]]
; COMPARE-LAA-MV: [[LATCH]]:
-; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP53:![0-9]+]]
-; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
entry:
@@ -4201,7 +4039,7 @@ define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP53:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-NO-MV: [[EXIT]]:
@@ -4238,7 +4076,7 @@ define void @stride_poison(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP53:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[EXIT]]:
@@ -4271,48 +4109,44 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-LABEL: define void @basic_strided_store(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]]
; COMPARE-NO-MV-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[WIDE_LOAD]], i64 0
-; COMPARE-NO-MV-NEXT: store i64 [[TMP6]], ptr [[TMP10]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[WIDE_LOAD]], i64 1
-; COMPARE-NO-MV-NEXT: store i64 [[TMP7]], ptr [[TMP11]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = extractelement <4 x i64> [[WIDE_LOAD]], i64 2
-; COMPARE-NO-MV-NEXT: store i64 [[TMP8]], ptr [[TMP12]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = extractelement <4 x i64> [[WIDE_LOAD]], i64 3
-; COMPARE-NO-MV-NEXT: store i64 [[TMP9]], ptr [[TMP13]], align 8
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP55:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @basic_strided_store(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -4324,7 +4158,7 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: store <4 x i64> [[WIDE_LOAD]], ptr [[TMP1]], align 8
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP55:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP54:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -4338,7 +4172,7 @@ define void @basic_strided_store(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IDX]]
; COMPARE-LAA-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP56:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP55:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -4370,47 +4204,51 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; COMPARE-NO-MV-LABEL: define void @ptr_vec_use(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr noalias [[P_PTR_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P]], <4 x i64> [[TMP0]]
; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x ptr> [[TMP1]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = load i64, ptr [[TMP2]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x ptr> [[TMP1]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = load i64, ptr [[TMP3]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x ptr> [[TMP1]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP4]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x ptr> [[TMP1]], i64 3
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = insertelement <4 x i64> poison, i64 [[TMP6]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = insertelement <4 x i64> [[TMP10]], i64 [[TMP7]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = insertelement <4 x i64> [[TMP11]], i64 [[TMP8]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> [[TMP12]], i64 [[TMP9]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP13]], ptr [[TMP14]], align 8
; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = getelementptr ptr, ptr [[P_PTR_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-NEXT: store <4 x ptr> [[TMP1]], ptr [[TMP15]], align 8
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[TMP0]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP56:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[LD]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[GEP_PTR_ST:%.*]] = getelementptr ptr, ptr [[P_PTR_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store ptr [[GEP_LD]], ptr [[GEP_PTR_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP57:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @ptr_vec_use(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr noalias [[P_PTR_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -4427,7 +4265,7 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP57:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP56:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -4443,7 +4281,7 @@ define void @ptr_vec_use(ptr noalias %p.out, ptr noalias %p.ptr.out, ptr %p, i64
; COMPARE-LAA-MV-NEXT: [[GEP_PTR_ST:%.*]] = getelementptr ptr, ptr [[P_PTR_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store ptr [[GEP_LD]], ptr [[GEP_PTR_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP58:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP57:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -4478,49 +4316,48 @@ define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-NO-MV-LABEL: define void @stride_idx_vec_use(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i64 0
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP1]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP0]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP0]], i64 3
+; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP5]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = load <4 x i64>, ptr [[TMP8]], align 8
; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = mul <4 x i64> [[TMP16]], [[TMP0]]
-; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
+; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[TMP4]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP17]], ptr [[TMP18]], align 8
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP4]], 4
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[TMP0]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP47:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP19]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP58:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[VAL:%.*]] = mul i64 [[LD]], [[IDX]]
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP59:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @stride_idx_vec_use(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -4535,7 +4372,7 @@ define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP59:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP58:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -4550,7 +4387,7 @@ define void @stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride) {
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP60:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP59:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -4583,50 +4420,51 @@ define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-NO-MV-LABEL: define void @offset_stride_idx_vec_use(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[STRIDE]], i64 0
-; COMPARE-NO-MV-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; COMPARE-NO-MV-NEXT: br i1 [[STRIDES_MV_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH1:.*]]
+; COMPARE-NO-MV: [[VECTOR_PH1]]:
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = mul <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = add <4 x i64> [[TMP0]], splat (i64 42)
; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i64 0
; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; COMPARE-NO-MV-NEXT: [[TMP3:%.*]] = extractelement <4 x i64> [[TMP1]], i64 1
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP3]]
-; COMPARE-NO-MV-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i64 2
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP4]]
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP1]], i64 3
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP5]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = load i64, ptr [[TMP6]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = load i64, ptr [[TMP7]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
-; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
-; COMPARE-NO-MV-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
-; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
+; COMPARE-NO-MV-NEXT: [[TMP17:%.*]] = load <4 x i64>, ptr [[TMP6]], align 8
; COMPARE-NO-MV-NEXT: [[TMP18:%.*]] = mul <4 x i64> [[TMP17]], [[TMP1]]
; COMPARE-NO-MV-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[INDEX]]
; COMPARE-NO-MV-NEXT: store <4 x i64> [[TMP18]], ptr [[TMP19]], align 8
; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
+; COMPARE-NO-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[TMP0]], splat (i64 4)
; COMPARE-NO-MV-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP60:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
; COMPARE-NO-MV-NEXT: br label %[[EXIT:.*]]
+; COMPARE-NO-MV: [[SCALAR_PH]]:
+; COMPARE-NO-MV-NEXT: br label %[[HEADER:.*]]
+; COMPARE-NO-MV: [[HEADER]]:
+; COMPARE-NO-MV-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[HEADER]] ]
+; COMPARE-NO-MV-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; COMPARE-NO-MV-NEXT: [[IV_TIMES_STRIDE:%.*]] = mul i64 [[IV]], [[STRIDE]]
+; COMPARE-NO-MV-NEXT: [[IDX:%.*]] = add i64 [[IV_TIMES_STRIDE]], 42
+; COMPARE-NO-MV-NEXT: [[GEP_LD:%.*]] = getelementptr i64, ptr [[P]], i64 [[IDX]]
+; COMPARE-NO-MV-NEXT: [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
+; COMPARE-NO-MV-NEXT: [[VAL:%.*]] = mul i64 [[LD]], [[IDX]]
+; COMPARE-NO-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
+; COMPARE-NO-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
+; COMPARE-NO-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
+; COMPARE-NO-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP61:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
; COMPARE-LAA-MV-LABEL: define void @offset_stride_idx_vec_use(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -4643,7 +4481,7 @@ define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[VEC_IND_NEXT]] = add nsw <4 x i64> [[VEC_IND]], splat (i64 4)
; COMPARE-LAA-MV-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP61:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP60:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[EXIT:.*]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -4659,7 +4497,7 @@ define void @offset_stride_idx_vec_use(ptr noalias %p.out, ptr %p, i64 %stride)
; COMPARE-LAA-MV-NEXT: [[GEP_ST:%.*]] = getelementptr i64, ptr [[P_OUT]], i64 [[IV]]
; COMPARE-LAA-MV-NEXT: store i64 [[VAL]], ptr [[GEP_ST]], align 8
; COMPARE-LAA-MV-NEXT: [[EXITCOND:%.*]] = icmp slt i64 [[IV_NEXT]], 128
-; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP62:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EXITCOND]], label %[[HEADER]], label %[[EXIT]], !llvm.loop [[LOOP61:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
@@ -4755,43 +4593,26 @@ exit:
define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; COMPARE-NO-MV-LABEL: define void @test_rewrite_iv_scevs(
; COMPARE-NO-MV-SAME: i32 [[START:%.*]], ptr [[DST:%.*]]) {
-; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
+; COMPARE-NO-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-NO-MV-NEXT: [[START_EXT:%.*]] = zext i32 [[START]] to i64
-; COMPARE-NO-MV-NEXT: [[TMP0:%.*]] = sub nsw i64 100, [[START_EXT]]
-; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
+; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ne i32 [[START]], 1
+; COMPARE-NO-MV-NEXT: br label %[[STRIDES_CHECK:.*]]
+; COMPARE-NO-MV: [[STRIDES_CHECK]]:
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
-; COMPARE-NO-MV-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
-; COMPARE-NO-MV-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = add i64 [[START_EXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: [[TMP2:%.*]] = mul i64 [[N_VEC]], [[START_EXT]]
; COMPARE-NO-MV-NEXT: br label %[[VECTOR_BODY:.*]]
; COMPARE-NO-MV: [[VECTOR_BODY]]:
-; COMPARE-NO-MV-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COMPARE-NO-MV-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], [[START_EXT]]
-; COMPARE-NO-MV-NEXT: [[TMP5:%.*]] = mul i64 1, [[START_EXT]]
-; COMPARE-NO-MV-NEXT: [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], [[TMP5]]
-; COMPARE-NO-MV-NEXT: [[TMP7:%.*]] = mul i64 2, [[START_EXT]]
-; COMPARE-NO-MV-NEXT: [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], [[TMP7]]
-; COMPARE-NO-MV-NEXT: [[TMP9:%.*]] = mul i64 3, [[START_EXT]]
-; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = add i64 [[OFFSET_IDX]], [[TMP9]]
-; COMPARE-NO-MV-NEXT: [[TMP11:%.*]] = getelementptr float, ptr [[DST]], i64 [[OFFSET_IDX]]
-; COMPARE-NO-MV-NEXT: [[TMP12:%.*]] = getelementptr float, ptr [[DST]], i64 [[TMP6]]
-; COMPARE-NO-MV-NEXT: [[TMP13:%.*]] = getelementptr float, ptr [[DST]], i64 [[TMP8]]
+; COMPARE-NO-MV-NEXT: [[TMP10:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COMPARE-NO-MV-NEXT: [[TMP14:%.*]] = getelementptr float, ptr [[DST]], i64 [[TMP10]]
-; COMPARE-NO-MV-NEXT: store float 0.000000e+00, ptr [[TMP11]], align 4
-; COMPARE-NO-MV-NEXT: store float 0.000000e+00, ptr [[TMP12]], align 4
-; COMPARE-NO-MV-NEXT: store float 0.000000e+00, ptr [[TMP13]], align 4
-; COMPARE-NO-MV-NEXT: store float 0.000000e+00, ptr [[TMP14]], align 4
-; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; COMPARE-NO-MV-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]]
+; COMPARE-NO-MV-NEXT: store <4 x float> zeroinitializer, ptr [[TMP14]], align 4
+; COMPARE-NO-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP10]], 4
+; COMPARE-NO-MV-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
+; COMPARE-NO-MV-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP62:![0-9]+]]
; COMPARE-NO-MV: [[MIDDLE_BLOCK]]:
-; COMPARE-NO-MV-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; COMPARE-NO-MV-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; COMPARE-NO-MV-NEXT: br label %[[SCALAR_PH]]
; COMPARE-NO-MV: [[SCALAR_PH]]:
-; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[START_EXT]], %[[ENTRY]] ]
-; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 97, %[[MIDDLE_BLOCK]] ], [ [[START_EXT]], %[[STRIDES_CHECK]] ]
+; COMPARE-NO-MV-NEXT: [[BC_RESUME_VAL1:%.*]] = phi i64 [ 96, %[[MIDDLE_BLOCK]] ], [ 0, %[[STRIDES_CHECK]] ]
; COMPARE-NO-MV-NEXT: br label %[[LOOP:.*]]
; COMPARE-NO-MV: [[LOOP]]:
; COMPARE-NO-MV-NEXT: [[IV_0:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_0_NEXT:%.*]], %[[LOOP]] ]
@@ -4801,7 +4622,7 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; COMPARE-NO-MV-NEXT: [[IV_1_NEXT]] = add i64 [[IV_1]], [[START_EXT]]
; COMPARE-NO-MV-NEXT: [[IV_0_NEXT]] = add i64 [[IV_0]], 1
; COMPARE-NO-MV-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_0_NEXT]], 100
-; COMPARE-NO-MV-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP50:![0-9]+]]
+; COMPARE-NO-MV-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP63:![0-9]+]]
; COMPARE-NO-MV: [[EXIT]]:
; COMPARE-NO-MV-NEXT: ret void
;
@@ -4809,10 +4630,9 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; COMPARE-LAA-MV-SAME: i32 [[START:%.*]], ptr [[DST:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*:]]
; COMPARE-LAA-MV-NEXT: [[START_EXT:%.*]] = zext i32 [[START]] to i64
-; COMPARE-LAA-MV-NEXT: [[TMP0:%.*]] = sub i64 100, [[START_EXT]]
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[START]], 1
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[START]], 1
; COMPARE-LAA-MV-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-LAA-MV: [[VECTOR_PH]]:
; COMPARE-LAA-MV-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -4822,7 +4642,7 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; COMPARE-LAA-MV-NEXT: store <4 x float> zeroinitializer, ptr [[TMP1]], align 4
; COMPARE-LAA-MV-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; COMPARE-LAA-MV-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP63:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP62:![0-9]+]]
; COMPARE-LAA-MV: [[MIDDLE_BLOCK]]:
; COMPARE-LAA-MV-NEXT: br label %[[SCALAR_PH]]
; COMPARE-LAA-MV: [[SCALAR_PH]]:
@@ -4837,7 +4657,7 @@ define void @test_rewrite_iv_scevs(i32 %start, ptr %dst) {
; COMPARE-LAA-MV-NEXT: [[IV_1_NEXT]] = add i64 [[IV_1]], [[START_EXT]]
; COMPARE-LAA-MV-NEXT: [[IV_0_NEXT]] = add i64 [[IV_0]], 1
; COMPARE-LAA-MV-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_0_NEXT]], 100
-; COMPARE-LAA-MV-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP64:![0-9]+]]
+; COMPARE-LAA-MV-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP63:![0-9]+]]
; COMPARE-LAA-MV: [[EXIT]]:
; COMPARE-LAA-MV-NEXT: ret void
;
>From 9e7830d8a506466acf312539ce6ccf0df37d0f74 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Thu, 16 Jul 2026 16:54:18 -0700
Subject: [PATCH 2/3] Address review comments
---
.../Transforms/Vectorize/LoopVectorize.cpp | 2 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 43 ++++++++-----------
.../Transforms/Vectorize/VPlanTransforms.h | 9 ++--
3 files changed, 24 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 118d4402cbf37..b3e261597c9cf 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6741,7 +6741,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
OrigLoop);
RUN_VPLAN_PASS(VPlanTransforms::makeMemOpWideningDecisions, *Plan, Range,
- RecipeBuilder, CostCtx);
+ RecipeBuilder, CostCtx, Config);
RUN_VPLAN_PASS(VPlanTransforms::makeScalarizationDecisions, *Plan, Range);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 6d556a3bd29be..3895520253732 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -7348,7 +7348,8 @@ static std::optional<int64_t> getConstantStride(VPValue *Addr, Type *AccessTy,
void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
VPRecipeBuilder &RecipeBuilder,
- VPCostContext &CostCtx) {
+ VPCostContext &CostCtx,
+ VFSelectionContext &Config) {
// Collect all loads/stores first. We will start with ones having simpler
// decisions followed by more complex ones that are potentially
// guided/dependent on the simpler ones.
@@ -7494,7 +7495,7 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
if (EnableVPlanBasedStrideMV)
RUN_VPLAN_PASS(VPlanTransforms::multiversionForUnitStridedMemOps, Plan,
- CostCtx, RecipeBuilder, Range, MemOps);
+ CostCtx, Config, RecipeBuilder, Range, MemOps);
VPlanTransforms::runPass("delegateMemOpWideningToLegacyCM", ProcessSubset,
Plan, [&](VPInstruction *VPI) {
@@ -7507,8 +7508,9 @@ void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
}
void VPlanTransforms::multiversionForUnitStridedMemOps(
- VPlan &Plan, VPCostContext &CostCtx, VPRecipeBuilder &RecipeBuilder,
- VFRange &Range, SmallVectorImpl<VPInstruction *> &MemOps) {
+ VPlan &Plan, VPCostContext &CostCtx, VFSelectionContext &Config,
+ VPRecipeBuilder &RecipeBuilder, VFRange &Range,
+ SmallVectorImpl<VPInstruction *> &MemOps) {
if (CostCtx.L->getHeader()->getParent()->hasOptSize())
return;
SmallVector<VPInstruction *> RemainingOps;
@@ -7529,38 +7531,28 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
const SCEV *PtrSCEV =
vputils::getSCEVExprForVPValue(PtrOp, CostCtx.PSE, CostCtx.L);
- const SCEV *Start = nullptr;
- const SCEV *Stride = nullptr;
+ const SCEV *Start, *Stride;
if (!match(PtrSCEV, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Stride),
- m_SpecificLoop(CostCtx.L)))) {
+ m_SpecificLoop(CostCtx.L))))
return Skip();
- }
Type *ScalarTy = VPI->getOpcode() == Instruction::Load
? VPI->getScalarType()
: VPI->getOperand(0)->getScalarType();
if (VPI->getMask()) {
- auto &TTI = CostCtx.TTI;
Instruction *I = VPI->getUnderlyingInstr();
- unsigned AS = getLoadStoreAddressSpace(I);
- const Align Alignment = getLoadStoreAlignment(I);
if (!LoopVectorizationPlanner::getDecisionAndClampRange(
[&](ElementCount VF) -> bool {
- Type *VTy = VectorType::get(ScalarTy, VF);
- return VPI->getOpcode() == Instruction::Load
- ? (TTI.isLegalMaskedLoad(VTy, Alignment, AS) ||
- TTI.isLegalMaskedGather(VTy, Alignment))
- : (TTI.isLegalMaskedStore(VTy, Alignment, AS) ||
- TTI.isLegalMaskedScatter(VTy, Alignment));
+ return Config.isLegalMaskedLoadOrStore(I, VF);
},
Range))
return Skip();
}
const auto *TypeSize = cast<SCEVConstant>(SE->getSizeOfExpr(
- Stride->getType(), SE->getDataLayout().getTypeStoreSize(ScalarTy)));
+ Stride->getType(), SE->getDataLayout().getTypeAllocSize(ScalarTy)));
auto ReplaceWithUnitStrided = [&]() {
VPBuilder Builder(VPI);
@@ -7619,6 +7611,9 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
MVConst = SE->getTruncateOrSignExtend(MVConst, ToMultiVersion->getType());
}
+ if (match(ToMultiVersion, m_scev_UndefOrPoison()))
+ return Skip();
+
if (!isa<SCEVUnknown>(ToMultiVersion)) {
// Match legacy behavior.
// If/when changed, make sure that explicit poison/undef in the defining
@@ -7628,16 +7623,11 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
Value *StrideVal = cast<SCEVUnknown>(ToMultiVersion)->getValue();
- if (isa<UndefValue>(StrideVal))
- return Skip();
-
const SCEVPredicate *NewPred =
SE->getComparePredicate(CmpInst::ICMP_EQ, ToMultiVersion, MVConst);
// Check if new predicate implies that backedge is never taken. If so, there
// is no reason to multiversion for it.
- SmallVector<const SCEVPredicate *> Preds{&CostCtx.PSE.getPredicate(),
- &StridePredicates, NewPred};
auto *PredicatedMaxBTC = SE->rewriteUsingPredicate(
SE->getSymbolicMaxBackedgeTakenCount(CostCtx.L), CostCtx.L,
StridePredicates.getUnionWith(NewPred, *SE)
@@ -7645,10 +7635,11 @@ void VPlanTransforms::multiversionForUnitStridedMemOps(
if (LoopVectorizationPlanner::getDecisionAndClampRange(
[&](ElementCount VF) {
- return SE->isKnownPositive(SE->getMinusSCEV(
+ return SE->isKnownPredicate(
+ CmpPredicate(ICmpInst::ICMP_SLT), PredicatedMaxBTC,
SE->getConstant(PredicatedMaxBTC->getType(),
- VF.isScalable() ? 1 : VF.getFixedValue() - 1),
- PredicatedMaxBTC));
+ VF.isScalable() ? 1
+ : VF.getFixedValue() - 1));
},
Range))
return Skip();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 5fe168e2dcc4c..50be723a47405 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -13,6 +13,7 @@
#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H
#define LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H
+#include "LoopVectorizationPlanner.h"
#include "VPlan.h"
#include "VPlanVerifier.h"
#include "llvm/ADT/STLFunctionalExtras.h"
@@ -583,13 +584,15 @@ struct VPlanTransforms {
/// recipes. Non load/store input instructions are left unchanged.
static void makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
VPRecipeBuilder &RecipeBuilder,
- VPCostContext &CostCtx);
+ VPCostContext &CostCtx,
+ VFSelectionContext &Config);
/// \p MemOps must be updated to contain ones that haven't been processed by
/// the pass.
static void multiversionForUnitStridedMemOps(
- VPlan &Plan, VPCostContext &CostCtx, VPRecipeBuilder &RecipeBuilder,
- VFRange &Range, SmallVectorImpl<VPInstruction *> &MemOps);
+ VPlan &Plan, VPCostContext &CostCtx, VFSelectionContext &Config,
+ VPRecipeBuilder &RecipeBuilder, VFRange &Range,
+ SmallVectorImpl<VPInstruction *> &MemOps);
/// Make VPlan-based scalarization decision prior to delegating to the ones
/// made by the legacy CM. Only transforms "usesFirstLaneOnly` def-use chains
>From 8258e553e421f0e731c5f2b0a7965254d3f534d0 Mon Sep 17 00:00:00 2001
From: Andrei Elovikov <andrei.elovikov at sifive.com>
Date: Fri, 17 Jul 2026 09:18:10 -0700
Subject: [PATCH 3/3] Expand to VPlan recipes instead of to LLVM IR
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 42 +++++++++----------
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 36 ++++++++++++++++
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 5 +++
.../LoopVectorize/vplan-based-stride-mv.ll | 8 ++--
4 files changed, 65 insertions(+), 26 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 3895520253732..68e9d080cb713 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5806,23 +5806,30 @@ void VPlanTransforms::expandSCEVsToVPInstructions(VPlan &Plan,
->getDebugLoc();
VPSCEVExpander Expander(Builder, SE, DL);
- // Expand VPExpandSCEVRecipes to VPInstructions using VPSCEVExpander. During
- // the transition, unsupported VPExpandSCEVRecipes are skipped and left for
- // late expansion.
+ // Expand VPExpandSCEVRecipes and VPExpandStridePredicatesRecipes to
+ // VPInstructions using VPSCEVExpander. During the transition, unsupported
+ // recipes are skipped and left for late expansion.
for (VPRecipeBase &R : make_early_inc_range(*Entry)) {
- auto *ExpSCEV = dyn_cast<VPExpandSCEVRecipe>(&R);
- if (!ExpSCEV || ExpSCEV->user_empty())
+ auto *Def = dyn_cast<VPSingleDefRecipe>(&R);
+ if (!Def || Def->user_empty())
+ continue;
+
+ auto *ExpStrides = dyn_cast<VPExpandStridePredicatesRecipe>(Def);
+ auto *ExpSCEV = dyn_cast<VPExpandSCEVRecipe>(Def);
+ if (!ExpStrides && !ExpSCEV)
continue;
- Builder.setInsertPoint(ExpSCEV);
- VPValue *Expanded = Expander.tryToExpand(ExpSCEV->getSCEV());
+ Builder.setInsertPoint(Def);
+ VPValue *Expanded =
+ ExpSCEV ? Expander.tryToExpand(ExpSCEV->getSCEV())
+ : Expander.tryToExpandPredicate(ExpStrides->getSCEVPredicate());
if (!Expanded)
continue;
- ExpSCEV->replaceAllUsesWith(Expanded);
+ Def->replaceAllUsesWith(Expanded);
// TripCount should not be used after expansion to VPInstructions. Reset to
// poison to avoid dangling references.
- if (Plan.getTripCount() == ExpSCEV)
- Plan.resetTripCount(Plan.getPoison(ExpSCEV->getScalarType()));
- ExpSCEV->eraseFromParent();
+ if (Plan.getTripCount() == Def)
+ Plan.resetTripCount(Plan.getPoison(Def->getScalarType()));
+ Def->eraseFromParent();
}
}
@@ -5835,17 +5842,6 @@ VPlanTransforms::expandSCEVs(VPlan &Plan, ScalarEvolution &SE) {
DenseMap<const SCEV *, Value *> ExpandedSCEVs;
// Expand remaining VPExpandSCEVRecipes to IR instructions using SCEVExpander.
for (VPRecipeBase &R : make_early_inc_range(*Entry)) {
- if (auto *ExpStrides = dyn_cast<VPExpandStridePredicatesRecipe>(&R)) {
- Value *Res = Expander.expandCodeForPredicate(
- ExpStrides->getSCEVPredicate(), EntryBB->getTerminator());
- Res->setName("strides.mv.check");
- VPValue *Exp = Plan.getOrAddLiveIn(Res);
-
- ExpStrides->replaceAllUsesWith(Exp);
- ExpStrides->eraseFromParent();
- continue;
- }
-
auto *ExpSCEV = dyn_cast<VPExpandSCEVRecipe>(&R);
if (!ExpSCEV)
continue;
@@ -5859,6 +5855,8 @@ VPlanTransforms::expandSCEVs(VPlan &Plan, ScalarEvolution &SE) {
Plan.resetTripCount(Exp);
ExpSCEV->eraseFromParent();
}
+ // NOTE: All current uses of VPExpandStridePredicatesRecipe are expected to be
+ // expandable via VPSCEVExpander, thus no handling here.
assert(none_of(*Entry,
IsaPred<VPExpandSCEVRecipe, VPExpandStridePredicatesRecipe>) &&
"all VPExpandSCEVRecipes/VPExpandStridePredicatesRecipe must have "
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 98994291779b1..1d92873883e3d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -893,3 +893,39 @@ VPValue *VPSCEVExpander::tryToExpand(const SCEV *S) {
return nullptr;
}
}
+
+VPValue *VPSCEVExpander::tryToExpandPredicate(const SCEVPredicate *Pred) {
+ if (auto *CmpPred = dyn_cast<SCEVComparePredicate>(Pred)) {
+ VPValue *LHS = tryToExpand(CmpPred->getLHS());
+ if (!LHS)
+ return nullptr;
+ VPValue *RHS = tryToExpand(CmpPred->getRHS());
+ if (!RHS)
+ return nullptr;
+ // Match SCEVExpander::expandComparePredicate behavior:
+ auto InvPred = CmpInst::getInversePredicate(CmpPred->getPredicate());
+ return Builder.createICmp(InvPred, LHS, RHS, DL);
+ }
+
+ if (auto *UnionPred = dyn_cast<SCEVUnionPredicate>(Pred)) {
+ SmallVector<VPValue *, 4> Checks;
+ for (const auto *Pred : UnionPred->getPredicates()) {
+ VPValue *Check = tryToExpandPredicate(Pred);
+ if (!Check)
+ return nullptr;
+ Checks.push_back(Check);
+ }
+
+ if (Checks.empty())
+ return Builder.getPlan().getOrAddLiveIn(
+ ConstantInt::getFalse(Builder.getPlan().getContext()));
+
+ VPValue *Result = Checks[0];
+ for (VPValue *Check : ArrayRef<VPValue *>(Checks).drop_front())
+ Result = Builder.createOr(Result, Check, DL);
+ return Result;
+ }
+
+ assert(false && "We shouldn't be generating that kind of predicate!");
+ return nullptr;
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 2980b704ec8da..649efaadda163 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -189,6 +189,11 @@ class VPSCEVExpander {
/// Try to expand \p S into recipes and live-ins using the builder. Returns
/// nullptr if \p S cannot be expanded yet.
VPValue *tryToExpand(const SCEV *S);
+
+ /// Try to expand \p Pred into recipes and live-ins using the builder. Returns
+ /// nullptr if \p Pred cannot be expanded (not expected to happen for the
+ /// predicates we currently generate).
+ VPValue *tryToExpandPredicate(const SCEVPredicate *Pred);
};
//===----------------------------------------------------------------------===//
// Utilities for modifying predecessors and successors of VPlan blocks.
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
index 1c9931e3e881b..6fccb632640fa 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-based-stride-mv.ll
@@ -2280,8 +2280,8 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-NO-MV-LABEL: define void @non_constant_btc(
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
-; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
@@ -2321,8 +2321,8 @@ define void @non_constant_btc(ptr noalias %p.out, ptr %p, i64 %stride, i64 %n) {
; COMPARE-LAA-MV-LABEL: define void @non_constant_btc(
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-LAA-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
@@ -2670,8 +2670,8 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-NO-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) {
; COMPARE-NO-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-NO-MV-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
-; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[M]], i64 1)
+; COMPARE-NO-MV-NEXT: [[STRIDES_MV_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-NO-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
; COMPARE-NO-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; COMPARE-NO-MV: [[VECTOR_PH]]:
@@ -2712,8 +2712,8 @@ define void @stride_btc_checks_order(ptr noalias %p.out, ptr %p, i64 %stride, i6
; COMPARE-LAA-MV-SAME: ptr noalias [[P_OUT:%.*]], ptr [[P:%.*]], i64 [[STRIDE:%.*]], i64 [[M:%.*]]) {
; COMPARE-LAA-MV-NEXT: [[ENTRY:.*]]:
; COMPARE-LAA-MV-NEXT: [[N:%.*]] = mul i64 [[M]], [[STRIDE]]
-; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: [[SMAX1:%.*]] = call i64 @llvm.smax.i64(i64 [[M]], i64 1)
+; COMPARE-LAA-MV-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE]], 1
; COMPARE-LAA-MV-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX1]], 4
; COMPARE-LAA-MV-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
; COMPARE-LAA-MV: [[VECTOR_SCEVCHECK]]:
More information about the llvm-branch-commits
mailing list