[llvm] [LV] Strip VPlanNativePath (PR #189979)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 1 08:33:05 PDT 2026
https://github.com/artagnon created https://github.com/llvm/llvm-project/pull/189979
VPlanNativePath was introduced nearly a decade ago, with the intent of being temporary, but little progress has been made since. Much of it has bitrotted due to heavy changes to VPlan, and although it is a significant amount of code, the testing is very sparse: there are a total of 26 tests that even enable the VPlan-native path.
The big-ticket item for getting VPlanNativePath to work is to have outer loops supported by a dependence analysis -- currently, LoopAccessAnalysis bails out on non-innermost loops altogether.
Although there is undoubedly benefit to vectorizing outer loops, it is perhaps better to start attacking the problem from a clean slate, rather than trying to work with the broken VPlanNativePath.
Strip VPlanNativePath entirely, migrating some of the test-coverage to the usual innermost-loop path.
>From 705ac4d3022473a4edbd234ecef37aecd6007848 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Wed, 1 Apr 2026 15:43:22 +0100
Subject: [PATCH] [LV] Strip VPlanNativePath
VPlanNativePath was introduced nearly a decade ago, with the intent of
being temporary, but little progress has been made since. Much of it has
bitrotted due to heavy changes to VPlan, and although it is a
significant amount of code, the testing is very sparse: there are a
total of 26 tests that even enable the VPlan-native path.
The big-ticket item for getting VPlanNativePath to work is to have outer
loops supported by a dependence analysis -- currently,
LoopAccessAnalysis bails out on non-innermost loops altogether.
Although there is undoubedly benefit to vectorizing outer loops, it is
perhaps better to start attacking the problem from a clean slate, rather
than trying to work with the broken VPlanNativePath.
Strip VPlanNativePath entirely, migrating some of the test-coverage to
the usual innermost-loop path.
---
.../Vectorize/LoopVectorizationLegality.h | 22 +-
.../Vectorize/LoopVectorizationLegality.cpp | 179 +------
.../Vectorize/LoopVectorizationPlanner.h | 18 -
.../Transforms/Vectorize/LoopVectorize.cpp | 275 +---------
llvm/lib/Transforms/Vectorize/VPlan.cpp | 21 -
.../AArch64/outer_loop_prefer_scalable.ll | 106 ----
...outer_loop_test1_no_explicit_vect_width.ll | 151 ------
.../VPlan/vplan-printing-outer-loop.ll | 73 ---
.../VPlan/vplan-stress-test-no-explict-vf.ll | 43 --
.../VPlan/vplan_hcfg_stress_test.ll | 48 --
...outer_loop_test1_no_explicit_vect_width.ll | 115 -----
.../X86/vplan-native-inner-loop-only.ll | 83 ---
.../LoopVectorize/dbg-outer-loop-vect.ll | 172 -------
.../LoopVectorize/explicit_outer_detection.ll | 236 ---------
.../explicit_outer_nonuniform_inner.ll | 173 -------
.../explicit_outer_uniform_diverg_branch.ll | 136 -----
.../LoopVectorize/inner-loop-reduction.ll | 86 ++++
...-checks.ll => inner-loop-with-rtchecks.ll} | 2 +-
.../noalias-scope-decl-outer-loop.ll | 73 ---
.../non-widenable-intrinsics-outer-loop.ll | 477 ------------------
.../outer-loop-inner-latch-successors.ll | 210 --------
.../outer-loop-vec-phi-predecessor-order.ll | 121 -----
.../LoopVectorize/outer-loop-wide-phis.ll | 220 --------
.../LoopVectorize/outer_loop_early_exit.ll | 49 --
.../outer_loop_hcfg_construction.ll | 319 ------------
.../LoopVectorize/outer_loop_scalable.ll | 140 -----
.../LoopVectorize/outer_loop_test1.ll | 88 ----
.../LoopVectorize/outer_loop_test2.ll | 107 ----
...plan-outer-loop-uncomputable-trip-count.ll | 47 --
.../vplan-vectorize-inner-loop-reduction.ll | 78 ---
...iden-call-instruction.ll => widen-call.ll} | 67 +--
...-select-instruction.ll => widen-select.ll} | 158 +++---
32 files changed, 200 insertions(+), 3893 deletions(-)
delete mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-outer-loop.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/vplan-stress-test-no-explict-vf.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/vplan_hcfg_stress_test.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/X86/vplan-native-inner-loop-only.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/dbg-outer-loop-vect.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/explicit_outer_detection.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/explicit_outer_nonuniform_inner.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/inner-loop-reduction.ll
rename llvm/test/Transforms/LoopVectorize/{vplan-native-path-inner-loop-with-runtime-checks.ll => inner-loop-with-rtchecks.ll} (98%)
delete mode 100644 llvm/test/Transforms/LoopVectorize/noalias-scope-decl-outer-loop.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/outer-loop-wide-phis.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/outer_loop_early_exit.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/outer_loop_hcfg_construction.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/outer_loop_test2.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/vplan-outer-loop-uncomputable-trip-count.ll
delete mode 100644 llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll
rename llvm/test/Transforms/LoopVectorize/{vplan-widen-call-instruction.ll => widen-call.ll} (56%)
rename llvm/test/Transforms/LoopVectorize/{vplan-widen-select-instruction.ll => widen-select.ll} (52%)
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index cff9162e39f60..48a86cac58997 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -292,11 +292,7 @@ class LoopVectorizationLegality {
/// Returns true if it is legal to vectorize this loop.
/// This does not mean that it is profitable to vectorize this
/// loop, only that it is legal to do so.
- /// Temporarily taking UseVPlanNativePath parameter. If true, take
- /// the new code path being implemented for outer loop vectorization
- /// (should be functional for inner loop vectorization) based on VPlan.
- /// If false, good old LV code.
- bool canVectorize(bool UseVPlanNativePath);
+ bool canVectorize();
/// Returns true if it is legal to vectorize the FP math operations in this
/// loop. Vectorizing is legal if we allow reordering of FP operations, or if
@@ -500,11 +496,7 @@ class LoopVectorizationLegality {
/// Return true if the pre-header, exiting and latch blocks of \p Lp and all
/// its nested loops are considered legal for vectorization. These legal
/// checks are common for inner and outer loop vectorization.
- /// Temporarily taking UseVPlanNativePath parameter. If true, take
- /// the new code path being implemented for outer loop vectorization
- /// (should be functional for inner loop vectorization) based on VPlan.
- /// If false, good old LV code.
- bool canVectorizeLoopNestCFG(Loop *Lp, bool UseVPlanNativePath);
+ bool canVectorizeLoopNestCFG(Loop *Lp);
/// Set up outer loop inductions by checking Phis in outer loop header for
/// supported inductions (int inductions). Return false if any of these Phis
@@ -513,11 +505,7 @@ class LoopVectorizationLegality {
/// Return true if the pre-header, exiting and latch blocks of \p Lp
/// (non-recursive) are considered legal for vectorization.
- /// Temporarily taking UseVPlanNativePath parameter. If true, take
- /// the new code path being implemented for outer loop vectorization
- /// (should be functional for inner loop vectorization) based on VPlan.
- /// If false, good old LV code.
- bool canVectorizeLoopCFG(Loop *Lp, bool UseVPlanNativePath);
+ bool canVectorizeLoopCFG(Loop *Lp);
/// Check if a single basic block loop is vectorizable.
/// At this point we know that this is a loop with a constant trip count
@@ -542,10 +530,6 @@ class LoopVectorizationLegality {
/// transformation.
bool canVectorizeWithIfConvert();
- /// Return true if we can vectorize this outer loop. The method performs
- /// specific checks for outer loop vectorization.
- bool canVectorizeOuterLoop();
-
/// Returns true if this is an early exit loop that can be vectorized.
/// Currently, a loop with an uncountable early exit is considered
/// vectorizable if:
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 6e4ecbdecb7cf..e59c8e75ba16d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -335,86 +335,6 @@ void LoopVectorizeHints::setHint(StringRef Name, Metadata *Arg) {
}
}
-// Return true if the inner loop \p Lp is uniform with regard to the outer loop
-// \p OuterLp (i.e., if the outer loop is vectorized, all the vector lanes
-// executing the inner loop will execute the same iterations). This check is
-// very constrained for now but it will be relaxed in the future. \p Lp is
-// considered uniform if it meets all the following conditions:
-// 1) it has a canonical IV (starting from 0 and with stride 1),
-// 2) its latch terminator is a conditional branch and,
-// 3) its latch condition is a compare instruction whose operands are the
-// canonical IV and an OuterLp invariant.
-// This check doesn't take into account the uniformity of other conditions not
-// related to the loop latch because they don't affect the loop uniformity.
-//
-// NOTE: We decided to keep all these checks and its associated documentation
-// together so that we can easily have a picture of the current supported loop
-// nests. However, some of the current checks don't depend on \p OuterLp and
-// would be redundantly executed for each \p Lp if we invoked this function for
-// different candidate outer loops. This is not the case for now because we
-// don't currently have the infrastructure to evaluate multiple candidate outer
-// loops and \p OuterLp will be a fixed parameter while we only support explicit
-// outer loop vectorization. It's also very likely that these checks go away
-// before introducing the aforementioned infrastructure. However, if this is not
-// the case, we should move the \p OuterLp independent checks to a separate
-// function that is only executed once for each \p Lp.
-static bool isUniformLoop(Loop *Lp, Loop *OuterLp) {
- assert(Lp->getLoopLatch() && "Expected loop with a single latch.");
-
- // If Lp is the outer loop, it's uniform by definition.
- if (Lp == OuterLp)
- return true;
- assert(OuterLp->contains(Lp) && "OuterLp must contain Lp.");
-
- // 1.
- PHINode *IV = Lp->getCanonicalInductionVariable();
- if (!IV) {
- LLVM_DEBUG(dbgs() << "LV: Canonical IV not found.\n");
- return false;
- }
-
- // 2.
- BasicBlock *Latch = Lp->getLoopLatch();
- auto *LatchBr = dyn_cast<CondBrInst>(Latch->getTerminator());
- if (!LatchBr) {
- LLVM_DEBUG(dbgs() << "LV: Unsupported loop latch branch.\n");
- return false;
- }
-
- // 3.
- auto *LatchCmp = dyn_cast<CmpInst>(LatchBr->getCondition());
- if (!LatchCmp) {
- LLVM_DEBUG(
- dbgs() << "LV: Loop latch condition is not a compare instruction.\n");
- return false;
- }
-
- Value *CondOp0 = LatchCmp->getOperand(0);
- Value *CondOp1 = LatchCmp->getOperand(1);
- Value *IVUpdate = IV->getIncomingValueForBlock(Latch);
- if (!(CondOp0 == IVUpdate && OuterLp->isLoopInvariant(CondOp1)) &&
- !(CondOp1 == IVUpdate && OuterLp->isLoopInvariant(CondOp0))) {
- LLVM_DEBUG(dbgs() << "LV: Loop latch condition is not uniform.\n");
- return false;
- }
-
- return true;
-}
-
-// Return true if \p Lp and all its nested loops are uniform with regard to \p
-// OuterLp.
-static bool isUniformLoopNest(Loop *Lp, Loop *OuterLp) {
- if (!isUniformLoop(Lp, OuterLp))
- return false;
-
- // Check if nested loops are uniform.
- for (Loop *SubLp : *Lp)
- if (!isUniformLoopNest(SubLp, OuterLp))
- return false;
-
- return true;
-}
-
static IntegerType *getInductionIntegerTy(const DataLayout &DL, Type *Ty) {
assert(Ty->isIntOrPtrTy() && "Expected integer or pointer type");
@@ -630,73 +550,6 @@ bool LoopVectorizationLegality::isUniformMemOp(Instruction &I,
return isUniform(Ptr, VF) && !blockNeedsPredication(I.getParent());
}
-bool LoopVectorizationLegality::canVectorizeOuterLoop() {
- assert(!TheLoop->isInnermost() && "We are not vectorizing an outer loop.");
- // Store the result and return it at the end instead of exiting early, in case
- // allowExtraAnalysis is used to report multiple reasons for not vectorizing.
- bool Result = true;
- bool DoExtraAnalysis = ORE->allowExtraAnalysis(DEBUG_TYPE);
-
- for (BasicBlock *BB : TheLoop->blocks()) {
- // Check whether the BB terminator is a branch. Any other terminator is
- // not supported yet.
- Instruction *Term = BB->getTerminator();
- if (!isa<UncondBrInst, CondBrInst>(Term)) {
- reportVectorizationFailure("Unsupported basic block terminator",
- "loop control flow is not understood by vectorizer",
- "CFGNotUnderstood", ORE, TheLoop);
- if (DoExtraAnalysis)
- Result = false;
- else
- return false;
- }
-
- // Check whether the branch is a supported one. Only unconditional
- // branches, conditional branches with an outer loop invariant condition or
- // backedges are supported.
- // FIXME: We skip these checks when VPlan predication is enabled as we
- // want to allow divergent branches. This whole check will be removed
- // once VPlan predication is on by default.
- auto *Br = dyn_cast<CondBrInst>(Term);
- if (Br && !TheLoop->isLoopInvariant(Br->getCondition()) &&
- !LI->isLoopHeader(Br->getSuccessor(0)) &&
- !LI->isLoopHeader(Br->getSuccessor(1))) {
- reportVectorizationFailure("Unsupported conditional branch",
- "loop control flow is not understood by vectorizer",
- "CFGNotUnderstood", ORE, TheLoop);
- if (DoExtraAnalysis)
- Result = false;
- else
- return false;
- }
- }
-
- // Check whether inner loops are uniform. At this point, we only support
- // simple outer loops scenarios with uniform nested loops.
- if (!isUniformLoopNest(TheLoop /*loop nest*/,
- TheLoop /*context outer loop*/)) {
- reportVectorizationFailure("Outer loop contains divergent loops",
- "loop control flow is not understood by vectorizer",
- "CFGNotUnderstood", ORE, TheLoop);
- if (DoExtraAnalysis)
- Result = false;
- else
- return false;
- }
-
- // Check whether we are able to set up outer loop induction.
- if (!setupOuterLoopInductions()) {
- reportVectorizationFailure("Unsupported outer loop Phi(s)",
- "UnsupportedPhi", ORE, TheLoop);
- if (DoExtraAnalysis)
- Result = false;
- else
- return false;
- }
-
- return Result;
-}
-
void LoopVectorizationLegality::addInductionPhi(
PHINode *Phi, const InductionDescriptor &ID,
SmallPtrSetImpl<Value *> &AllowedExit) {
@@ -1616,10 +1469,8 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
}
// Helper function to canVectorizeLoopNestCFG.
-bool LoopVectorizationLegality::canVectorizeLoopCFG(Loop *Lp,
- bool UseVPlanNativePath) {
- assert((UseVPlanNativePath || Lp->isInnermost()) &&
- "VPlan-native path is not enabled.");
+bool LoopVectorizationLegality::canVectorizeLoopCFG(Loop *Lp) {
+ assert(Lp->isInnermost() && "Expected innermost loop");
// TODO: ORE should be improved to show more accurate information when an
// outer loop can't be vectorized because a nested loop is not understood or
@@ -1670,13 +1521,12 @@ bool LoopVectorizationLegality::canVectorizeLoopCFG(Loop *Lp,
return Result;
}
-bool LoopVectorizationLegality::canVectorizeLoopNestCFG(
- Loop *Lp, bool UseVPlanNativePath) {
+bool LoopVectorizationLegality::canVectorizeLoopNestCFG(Loop *Lp) {
// Store the result and return it at the end instead of exiting early, in case
// allowExtraAnalysis is used to report multiple reasons for not vectorizing.
bool Result = true;
bool DoExtraAnalysis = ORE->allowExtraAnalysis(DEBUG_TYPE);
- if (!canVectorizeLoopCFG(Lp, UseVPlanNativePath)) {
+ if (!canVectorizeLoopCFG(Lp)) {
if (DoExtraAnalysis)
Result = false;
else
@@ -1686,7 +1536,7 @@ bool LoopVectorizationLegality::canVectorizeLoopNestCFG(
// Recursively check whether the loop control flow of nested loops is
// understood.
for (Loop *SubLp : *Lp)
- if (!canVectorizeLoopNestCFG(SubLp, UseVPlanNativePath)) {
+ if (!canVectorizeLoopNestCFG(SubLp)) {
if (DoExtraAnalysis)
Result = false;
else
@@ -1930,7 +1780,7 @@ bool LoopVectorizationLegality::canUncountableExitConditionLoadBeMoved(
return true;
}
-bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
+bool LoopVectorizationLegality::canVectorize() {
// Store the result and return it at the end instead of exiting early, in case
// allowExtraAnalysis is used to report multiple reasons for not vectorizing.
bool Result = true;
@@ -1938,7 +1788,7 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
bool DoExtraAnalysis = ORE->allowExtraAnalysis(DEBUG_TYPE);
// Check whether the loop-related control flow in the loop nest is expected by
// vectorizer.
- if (!canVectorizeLoopNestCFG(TheLoop, UseVPlanNativePath)) {
+ if (!canVectorizeLoopNestCFG(TheLoop)) {
if (DoExtraAnalysis) {
LLVM_DEBUG(dbgs() << "LV: legality check failed: loop nest");
Result = false;
@@ -1954,18 +1804,9 @@ bool LoopVectorizationLegality::canVectorize(bool UseVPlanNativePath) {
// Specific checks for outer loops. We skip the remaining legal checks at this
// point because they don't support outer loops.
if (!TheLoop->isInnermost()) {
- assert(UseVPlanNativePath && "VPlan-native path is not enabled.");
-
- if (!canVectorizeOuterLoop()) {
- reportVectorizationFailure("Unsupported outer loop",
- "UnsupportedOuterLoop", ORE, TheLoop);
- // TODO: Implement DoExtraAnalysis when subsequent legal checks support
- // outer loops.
- return false;
- }
-
- LLVM_DEBUG(dbgs() << "LV: We can vectorize this outer loop!\n");
- return Result;
+ reportVectorizationFailure("Unsupported outer loop vectorization",
+ "UnsupportedOuterLoop", ORE, TheLoop);
+ return false;
}
assert(TheLoop->isInnermost() && "Inner loop expected.");
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index bab9efee37533..054184ea8e4eb 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -48,7 +48,6 @@ class VPRecipeBuilder;
struct VPRegisterUsage;
struct VFRange;
-extern cl::opt<bool> EnableVPlanNativePath;
extern cl::opt<unsigned> ForceTargetInstructionCost;
/// VPlan-based builder utility analogous to IRBuilder.
@@ -553,10 +552,6 @@ class LoopVectorizationPlanner {
/// interleaving should be avoided up-front, no plans are generated.
void plan(ElementCount UserVF, unsigned UserIC);
- /// Use the VPlan-native path to plan how to best vectorize, return the best
- /// VF and its cost.
- VectorizationFactor planInVPlanNativePath(ElementCount UserVF);
-
/// Return the VPlan for \p VF. At the moment, there is always a single VPlan
/// for each VF.
VPlan &getPlanFor(ElementCount VF) const;
@@ -643,20 +638,7 @@ class LoopVectorizationPlanner {
unsigned OrigLoopInvocationWeight, unsigned EstimatedVFxUF,
bool DisableRuntimeUnroll);
-protected:
- /// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
- /// according to the information gathered by Legal when it checked if it is
- /// legal to vectorize the loop.
- void buildVPlans(ElementCount MinVF, ElementCount MaxVF);
-
private:
- /// Build a VPlan according to the information gathered by Legal. \return a
- /// VPlan for vectorization factors \p Range.Start and up to \p Range.End
- /// exclusive, possibly decreasing \p Range.End. If no VPlan can be built for
- /// the input range, set the largest included VF to the maximum VF for which
- /// no plan could be built.
- VPlanPtr tryToBuildVPlan(VFRange &Range);
-
/// Build a VPlan using VPRecipes according to the information gather by
/// Legal. This method is only used for the legacy inner loop vectorizer.
/// \p Range's largest included VF is restricted to the maximum VF the
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 32ef10e530187..85ef646e09e09 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -26,12 +26,7 @@
// can be one, if vectorization is not profitable.
//
// There is a development effort going on to migrate loop vectorizer to the
-// VPlan infrastructure and to introduce outer loop vectorization support (see
-// docs/VectorizationPlan.rst and
-// http://lists.llvm.org/pipermail/llvm-dev/2017-December/119523.html). For this
-// purpose, we temporarily introduced the VPlan-native vectorization path: an
-// alternative vectorization path that is natively implemented on top of the
-// VPlan infrastructure. See EnableVPlanNativePath for enabling.
+// VPlan infrastructure and (see docs/VectorizationPlan.rst).
//
//===----------------------------------------------------------------------===//
//
@@ -355,11 +350,6 @@ static cl::opt<bool> PreferPredicatedReductionSelect(
cl::desc(
"Prefer predicating a reduction operation over an after loop select."));
-cl::opt<bool> llvm::EnableVPlanNativePath(
- "enable-vplan-native-path", cl::Hidden,
- cl::desc("Enable VPlan-native vectorization path with "
- "support for outer loop vectorization."));
-
cl::opt<bool>
llvm::VerifyEachVPlan("vplan-verify-each",
#ifdef EXPENSIVE_CHECKS
@@ -385,17 +375,6 @@ cl::opt<bool> llvm::VPlanPrintVectorRegionScope(
"`-vplan-print-after*` if the plan has one."));
#endif
-// This flag enables the stress testing of the VPlan H-CFG construction in the
-// VPlan-native vectorization path. It must be used in conjuction with
-// -enable-vplan-native-path. -vplan-verify-hcfg can also be used to enable the
-// verification of the H-CFGs built.
-static cl::opt<bool> VPlanBuildStressTest(
- "vplan-build-stress-test", cl::init(false), cl::Hidden,
- cl::desc(
- "Build VPlan for every supported loop nest in the function and bail "
- "out right after the build (stress test the VPlan H-CFG construction "
- "in the VPlan-native vectorization path)."));
-
cl::opt<bool> llvm::EnableLoopInterleaving(
"interleave-loops", cl::init(true), cl::Hidden,
cl::desc("Enable loop interleaving in Loop vectorization passes"));
@@ -1360,8 +1339,8 @@ class LoopVectorizationCostModel {
return;
// Override EVL styles if needed.
// FIXME: Investigate opportunity for fixed vector factor.
- bool EVLIsLegal = UserIC <= 1 && IsScalableVF &&
- TTI.hasActiveVectorLength() && !EnableVPlanNativePath;
+ bool EVLIsLegal =
+ UserIC <= 1 && IsScalableVF && TTI.hasActiveVectorLength();
if (EVLIsLegal)
return;
// If for some reason EVL mode is unsupported, fallback to a scalar epilogue
@@ -2100,48 +2079,6 @@ static bool useActiveLaneMaskForControlFlow(TailFoldingStyle Style) {
return Style == TailFoldingStyle::DataAndControlFlow;
}
-// Return true if \p OuterLp is an outer loop annotated with hints for explicit
-// vectorization. The loop needs to be annotated with #pragma omp simd
-// simdlen(#) or #pragma clang vectorize(enable) vectorize_width(#). If the
-// vector length information is not provided, vectorization is not considered
-// explicit. Interleave hints are not allowed either. These limitations will be
-// relaxed in the future.
-// Please, note that we are currently forced to abuse the pragma 'clang
-// vectorize' semantics. This pragma provides *auto-vectorization hints*
-// (i.e., LV must check that vectorization is legal) whereas pragma 'omp simd'
-// provides *explicit vectorization hints* (LV can bypass legal checks and
-// assume that vectorization is legal). However, both hints are implemented
-// using the same metadata (llvm.loop.vectorize, processed by
-// LoopVectorizeHints). This will be fixed in the future when the native IR
-// representation for pragma 'omp simd' is introduced.
-static bool isExplicitVecOuterLoop(Loop *OuterLp,
- OptimizationRemarkEmitter *ORE) {
- assert(!OuterLp->isInnermost() && "This is not an outer loop");
- LoopVectorizeHints Hints(OuterLp, true /*DisableInterleaving*/, *ORE);
-
- // Only outer loops with an explicit vectorization hint are supported.
- // Unannotated outer loops are ignored.
- if (Hints.getForce() == LoopVectorizeHints::FK_Undefined)
- return false;
-
- Function *Fn = OuterLp->getHeader()->getParent();
- if (!Hints.allowVectorization(Fn, OuterLp,
- true /*VectorizeOnlyWhenForced*/)) {
- LLVM_DEBUG(dbgs() << "LV: Loop hints prevent outer loop vectorization.\n");
- return false;
- }
-
- if (Hints.getInterleave() > 1) {
- // TODO: Interleave support is future work.
- LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for "
- "outer loops.\n");
- Hints.emitRemarkWithHints();
- return false;
- }
-
- return true;
-}
-
static void collectSupportedLoops(Loop &L, LoopInfo *LI,
OptimizationRemarkEmitter *ORE,
SmallVectorImpl<Loop *> &V) {
@@ -2149,8 +2086,7 @@ static void collectSupportedLoops(Loop &L, LoopInfo *LI,
// now, only collect outer loops that have explicit vectorization hints. If we
// are stress testing the VPlan H-CFG construction, we collect the outermost
// loop of every loop nest.
- if (L.isInnermost() || VPlanBuildStressTest ||
- (EnableVPlanNativePath && isExplicitVecOuterLoop(&L, ORE))) {
+ if (L.isInnermost()) {
LoopBlocksRPO RPOT(&L);
RPOT.perform(LI);
if (!containsIrreducibleCFG<const BasicBlock *>(RPOT, *LI)) {
@@ -6663,83 +6599,6 @@ void LoopVectorizationCostModel::collectInLoopReductions() {
}
}
-// This function will select a scalable VF if the target supports scalable
-// vectors and a fixed one otherwise.
-// TODO: we could return a pair of values that specify the max VF and
-// min VF, to be used in `buildVPlans(MinVF, MaxVF)` instead of
-// `buildVPlans(VF, VF)`. We cannot do it because VPLAN at the moment
-// doesn't have a cost model that can choose which plan to execute if
-// more than one is generated.
-static ElementCount determineVPlanVF(const TargetTransformInfo &TTI,
- LoopVectorizationCostModel &CM) {
- unsigned WidestType;
- std::tie(std::ignore, WidestType) = CM.getSmallestAndWidestTypes();
-
- TargetTransformInfo::RegisterKind RegKind =
- TTI.enableScalableVectorization()
- ? TargetTransformInfo::RGK_ScalableVector
- : TargetTransformInfo::RGK_FixedWidthVector;
-
- TypeSize RegSize = TTI.getRegisterBitWidth(RegKind);
- unsigned N = RegSize.getKnownMinValue() / WidestType;
- return ElementCount::get(N, RegSize.isScalable());
-}
-
-VectorizationFactor
-LoopVectorizationPlanner::planInVPlanNativePath(ElementCount UserVF) {
- ElementCount VF = UserVF;
- // Outer loop handling: They may require CFG and instruction level
- // transformations before even evaluating whether vectorization is profitable.
- // Since we cannot modify the incoming IR, we need to build VPlan upfront in
- // the vectorization pipeline.
- if (!OrigLoop->isInnermost()) {
- // If the user doesn't provide a vectorization factor, determine a
- // reasonable one.
- if (UserVF.isZero()) {
- VF = determineVPlanVF(TTI, CM);
- LLVM_DEBUG(dbgs() << "LV: VPlan computed VF " << VF << ".\n");
-
- // Make sure we have a VF > 1 for stress testing.
- if (VPlanBuildStressTest && (VF.isScalar() || VF.isZero())) {
- LLVM_DEBUG(dbgs() << "LV: VPlan stress testing: "
- << "overriding computed VF.\n");
- VF = ElementCount::getFixed(4);
- }
- } else if (UserVF.isScalable() && !TTI.supportsScalableVectors() &&
- !ForceTargetSupportsScalableVectors) {
- LLVM_DEBUG(dbgs() << "LV: Not vectorizing. Scalable VF requested, but "
- << "not supported by the target.\n");
- reportVectorizationFailure(
- "Scalable vectorization requested but not supported by the target",
- "the scalable user-specified vectorization width for outer-loop "
- "vectorization cannot be used because the target does not support "
- "scalable vectors.",
- "ScalableVFUnfeasible", ORE, OrigLoop);
- return VectorizationFactor::Disabled();
- }
- assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");
- assert(isPowerOf2_32(VF.getKnownMinValue()) &&
- "VF needs to be a power of two");
- LLVM_DEBUG(dbgs() << "LV: Using " << (!UserVF.isZero() ? "user " : "")
- << "VF " << VF << " to build VPlans.\n");
- buildVPlans(VF, VF);
-
- if (VPlans.empty())
- return VectorizationFactor::Disabled();
-
- // For VPlan build stress testing, we bail out after VPlan construction.
- if (VPlanBuildStressTest)
- return VectorizationFactor::Disabled();
-
- return {VF, 0 /*Cost*/, 0 /* ScalarCost */};
- }
-
- LLVM_DEBUG(
- dbgs() << "LV: Not vectorizing. Inner loops aren't supported in the "
- "VPlan-native path.\n");
- return VectorizationFactor::Disabled();
-}
-
void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
assert(OrigLoop->isInnermost() && "Inner loop expected.");
CM.collectValuesToIgnore();
@@ -8233,46 +8092,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
return Plan;
}
-VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
- // Outer loop handling: They may require CFG and instruction level
- // transformations before even evaluating whether vectorization is profitable.
- // Since we cannot modify the incoming IR, we need to build VPlan upfront in
- // the vectorization pipeline.
- assert(!OrigLoop->isInnermost());
- assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");
-
- auto Plan = VPlanTransforms::buildVPlan0(
- OrigLoop, *LI, Legal->getWidestInductionType(),
- getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE);
-
- VPlanTransforms::createHeaderPhiRecipes(
- *Plan, PSE, *OrigLoop, Legal->getInductionVars(),
- MapVector<PHINode *, RecurrenceDescriptor>(),
- SmallPtrSet<const PHINode *, 1>(), SmallPtrSet<PHINode *, 1>(),
- /*AllowReordering=*/false);
- [[maybe_unused]] bool CanHandleExits = VPlanTransforms::handleEarlyExits(
- *Plan, UncountableExitStyle::NoUncountableExit, OrigLoop, PSE, *DT,
- Legal->getAssumptionCache());
- assert(CanHandleExits &&
- "early-exits are not supported in VPlan-native path");
- VPlanTransforms::addMiddleCheck(*Plan, /*TailFolded*/ false);
-
- VPlanTransforms::createLoopRegions(*Plan);
-
- for (ElementCount VF : Range)
- Plan->addVF(VF);
-
- if (!VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(*Plan, *TLI))
- return nullptr;
-
- // Optimize induction live-out users to use precomputed end values.
- VPlanTransforms::optimizeInductionLiveOutUsers(*Plan, PSE,
- /*FoldTail=*/false);
-
- assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
- return Plan;
-}
-
void LoopVectorizationPlanner::addReductionResultComputation(
VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder, ElementCount MinVF) {
using namespace VPlanPatternMatch;
@@ -8477,7 +8296,7 @@ void LoopVectorizationPlanner::attachRuntimeChecks(
if (MemCheckBlock && MemCheckBlock->hasNPredecessors(0)) {
// VPlan-native path does not do any analysis for runtime checks
// currently.
- assert((!EnableVPlanNativePath || OrigLoop->isInnermost()) &&
+ assert(OrigLoop->isInnermost() &&
"Runtime checks are not supported for outer loops yet");
if (CM.OptForSize) {
@@ -8556,74 +8375,6 @@ static ScalarEpilogueLowering getScalarEpilogueLowering(
return CM_ScalarEpilogueAllowed;
}
-// Process the loop in the VPlan-native vectorization path. This path builds
-// VPlan upfront in the vectorization pipeline, which allows to apply
-// VPlan-to-VPlan transformations from the very beginning without modifying the
-// input LLVM IR.
-static bool processLoopInVPlanNativePath(
- Loop *L, PredicatedScalarEvolution &PSE, LoopInfo *LI, DominatorTree *DT,
- LoopVectorizationLegality *LVL, TargetTransformInfo *TTI,
- TargetLibraryInfo *TLI, DemandedBits *DB, AssumptionCache *AC,
- OptimizationRemarkEmitter *ORE,
- std::function<BlockFrequencyInfo &()> GetBFI, bool OptForSize,
- LoopVectorizeHints &Hints, LoopVectorizationRequirements &Requirements) {
-
- if (isa<SCEVCouldNotCompute>(PSE.getBackedgeTakenCount())) {
- LLVM_DEBUG(dbgs() << "LV: cannot compute the outer-loop trip count\n");
- return false;
- }
- assert(EnableVPlanNativePath && "VPlan-native path is disabled.");
- Function *F = L->getHeader()->getParent();
- InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL->getLAI());
-
- ScalarEpilogueLowering SEL =
- getScalarEpilogueLowering(F, L, Hints, OptForSize, TTI, TLI, *LVL, &IAI);
-
- LoopVectorizationCostModel CM(SEL, L, PSE, LI, LVL, *TTI, TLI, DB, AC, ORE,
- GetBFI, F, &Hints, IAI, OptForSize);
- // Use the planner for outer loop vectorization.
- // TODO: CM is not used at this point inside the planner. Turn CM into an
- // optional argument if we don't need it in the future.
- LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, LVL, CM, IAI, PSE, Hints,
- ORE);
-
- // Get user vectorization factor.
- ElementCount UserVF = Hints.getWidth();
-
- CM.collectElementTypesForWidening();
-
- // Plan how to best vectorize, return the best VF and its cost.
- const VectorizationFactor VF = LVP.planInVPlanNativePath(UserVF);
-
- // If we are stress testing VPlan builds, do not attempt to generate vector
- // code. Masked vector code generation support will follow soon.
- // Also, do not attempt to vectorize if no vector code will be produced.
- if (VPlanBuildStressTest || VectorizationFactor::Disabled() == VF)
- return false;
-
- VPlan &BestPlan = LVP.getPlanFor(VF.Width);
-
- {
- GeneratedRTChecks Checks(PSE, DT, LI, TTI, CM.CostKind);
- InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, /*UF=*/1, &CM,
- Checks, BestPlan);
- LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \""
- << L->getHeader()->getParent()->getName() << "\"\n");
- LVP.addMinimumIterationCheck(BestPlan, VF.Width, /*UF=*/1,
- VF.MinProfitableTripCount);
- bool HasBranchWeights =
- hasBranchWeightMD(*L->getLoopLatch()->getTerminator());
- LVP.attachRuntimeChecks(BestPlan, Checks, HasBranchWeights);
-
- LVP.executePlan(VF.Width, /*UF=*/1, BestPlan, LB, DT);
- }
-
- reportVectorization(ORE, L, VF, 1);
-
- assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs()));
- return true;
-}
-
// Emit a remark if there are stores to floats that required a floating point
// extension. If the vectorized loop was generated with floating point there
// will be a performance penalty from the conversion overhead and the change in
@@ -9248,7 +8999,7 @@ static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
}
bool LoopVectorizePass::processLoop(Loop *L) {
- assert((EnableVPlanNativePath || L->isInnermost()) &&
+ assert(L->isInnermost() &&
"VPlan-native path is not enabled. Only process inner loops.");
LLVM_DEBUG(dbgs() << "\nLV: Checking a loop in '"
@@ -9298,7 +9049,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
LoopVectorizationLegality LVL(L, PSE, DT, TTI, TLI, F, *LAIs, LI, ORE,
&Requirements, &Hints, DB, AC,
/*AllowRuntimeSCEVChecks=*/!OptForSize, AA);
- if (!LVL.canVectorize(EnableVPlanNativePath)) {
+ if (!LVL.canVectorize()) {
LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n");
Hints.emitRemarkWithHints();
return false;
@@ -9313,18 +9064,6 @@ bool LoopVectorizePass::processLoop(Loop *L) {
}
}
- // Entrance to the VPlan-native vectorization path. Outer loops are processed
- // here. They may require CFG and instruction level transformations before
- // even evaluating whether vectorization is profitable. Since we cannot modify
- // the incoming IR, we need to build VPlan upfront in the vectorization
- // pipeline.
- if (!L->isInnermost())
- return processLoopInVPlanNativePath(L, PSE, LI, DT, &LVL, TTI, TLI, DB, AC,
- ORE, GetBFI, OptForSize, Hints,
- Requirements);
-
- assert(L->isInnermost() && "Inner loop expected.");
-
InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL.getLAI());
bool UseInterleaved = TTI->enableInterleavedAccessVectorization();
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 919166099a4d1..f3bb26e08926b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1608,27 +1608,6 @@ bool LoopVectorizationPlanner::getDecisionAndClampRange(
return PredicateAtRangeStart;
}
-/// Build VPlans for the full range of feasible VF's = {\p MinVF, 2 * \p MinVF,
-/// 4 * \p MinVF, ..., \p MaxVF} by repeatedly building a VPlan for a sub-range
-/// of VF's starting at a given VF and extending it as much as possible. Each
-/// vectorization decision can potentially shorten this sub-range during
-/// buildVPlan().
-void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF,
- ElementCount MaxVF) {
- auto MaxVFTimes2 = MaxVF * 2;
- for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
- VFRange SubRange = {VF, MaxVFTimes2};
- if (auto Plan = tryToBuildVPlan(SubRange)) {
- VPlanTransforms::optimize(*Plan);
- // Update the name of the latch of the top-level vector loop region region
- // after optimizations which includes block folding.
- Plan->getVectorLoopRegion()->getExiting()->setName("vector.latch");
- VPlans.push_back(std::move(Plan));
- }
- VF = SubRange.End;
- }
-}
-
VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF) const {
assert(count_if(VPlans,
[VF](const VPlanPtr &Plan) { return Plan->hasVF(VF); }) ==
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
deleted file mode 100644
index e6be3414eb26a..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
+++ /dev/null
@@ -1,106 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
-; RUN: opt -S -mtriple aarch64 -mattr=+sve -passes=loop-vectorize -enable-vplan-native-path < %s | FileCheck %s
-
- at A = external global [1024 x float], align 4
- at B = external global [512 x float], align 4
-
-; Test if the vplan-native-path successfully vectorizes a loop using scalable vectors if the target preferes scalable vectors.
-define void @foo() {
-; CHECK-LABEL: define void @foo
-; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_LATCH:%.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_LATCH]] ]
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds [1024 x float], ptr @A, i64 0, <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP5]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> poison)
-; CHECK-NEXT: br label [[INNER_LOOP1:%.*]]
-; CHECK: inner_loop1:
-; CHECK-NEXT: [[TMP6:%.*]] = phi <vscale x 4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP10:%.*]], [[INNER_LOOP1]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = phi <vscale x 4 x float> [ [[WIDE_MASKED_GATHER]], [[VECTOR_BODY]] ], [ [[TMP9:%.*]], [[INNER_LOOP1]] ]
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [512 x float], ptr @B, i64 0, <vscale x 4 x i64> [[TMP6]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP8]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> poison)
-; CHECK-NEXT: [[TMP9]] = fmul <vscale x 4 x float> [[TMP7]], [[WIDE_MASKED_GATHER2]]
-; CHECK-NEXT: [[TMP10]] = add nuw nsw <vscale x 4 x i64> [[TMP6]], splat (i64 1)
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq <vscale x 4 x i64> [[TMP10]], splat (i64 512)
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <vscale x 4 x i1> [[TMP11]], i32 0
-; CHECK-NEXT: br i1 [[TMP12]], label [[VECTOR_LATCH]], label [[INNER_LOOP1]]
-; CHECK: vector.latch:
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> [[TMP9]], <vscale x 4 x ptr> align 4 [[TMP5]], <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT: br label [[OUTER_LOOP:%.*]]
-; CHECK: outer_loop:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[OUTER_LOOP_LATCH:%.*]] ]
-; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 [[I]]
-; CHECK-NEXT: [[X_START:%.*]] = load float, ptr [[ARRAYIDX1]], align 4
-; CHECK-NEXT: br label [[INNER_LOOP:%.*]]
-; CHECK: inner_loop:
-; CHECK-NEXT: [[J:%.*]] = phi i64 [ 0, [[OUTER_LOOP]] ], [ [[J_NEXT:%.*]], [[INNER_LOOP]] ]
-; CHECK-NEXT: [[X:%.*]] = phi float [ [[X_START]], [[OUTER_LOOP]] ], [ [[X_NEXT:%.*]], [[INNER_LOOP]] ]
-; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [512 x float], ptr @B, i64 0, i64 [[J]]
-; CHECK-NEXT: [[B:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
-; CHECK-NEXT: [[X_NEXT]] = fmul float [[X]], [[B]]
-; CHECK-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J]], 1
-; CHECK-NEXT: [[INNER_EXITCOND:%.*]] = icmp eq i64 [[J_NEXT]], 512
-; CHECK-NEXT: br i1 [[INNER_EXITCOND]], label [[OUTER_LOOP_LATCH]], label [[INNER_LOOP]]
-; CHECK: outer_loop_latch:
-; CHECK-NEXT: [[X_NEXT_LCSSA:%.*]] = phi float [ [[X_NEXT]], [[INNER_LOOP]] ]
-; CHECK-NEXT: store float [[X_NEXT_LCSSA]], ptr [[ARRAYIDX1]], align 4
-; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
-; CHECK-NEXT: [[OUTER_EXITCOND:%.*]] = icmp eq i64 [[I_NEXT]], 1024
-; CHECK-NEXT: br i1 [[OUTER_EXITCOND]], label [[EXIT]], label [[OUTER_LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: exit:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %outer_loop
-
-outer_loop:
- %i = phi i64 [ 0, %entry ], [ %i.next, %outer_loop_latch ]
- %arrayidx1 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %i
- %x.start = load float, ptr %arrayidx1, align 4
- br label %inner_loop
-
-inner_loop:
- %j = phi i64 [ 0, %outer_loop ], [ %j.next, %inner_loop ]
- %x = phi float [ %x.start, %outer_loop ], [ %x.next, %inner_loop ]
- %arrayidx2 = getelementptr inbounds [512 x float], ptr @B, i64 0, i64 %j
- %b = load float, ptr %arrayidx2, align 4
- %x.next = fmul float %x, %b
- %j.next = add nuw nsw i64 %j, 1
- %inner_exitcond = icmp eq i64 %j.next, 512
- br i1 %inner_exitcond, label %outer_loop_latch, label %inner_loop
-
-outer_loop_latch:
- store float %x.next, ptr %arrayidx1, align 4
- %i.next = add nuw nsw i64 %i, 1
- %outer_exitcond = icmp eq i64 %i.next, 1024
- br i1 %outer_exitcond, label %exit, label %outer_loop, !llvm.loop !1
-
-exit:
- ret void
-}
-
-!1 = distinct !{!1, !2}
-!2 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll
deleted file mode 100644
index 237867ed05f64..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll
+++ /dev/null
@@ -1,151 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 5
-; RUN: opt -S -passes=loop-vectorize -enable-vplan-native-path -mtriple aarch64-gnu-linux < %s | FileCheck %s
-
-; extern int arr[8][8];
-; extern int arr2[8];
-;
-; void foo(int n)
-; {
-; int i1, i2;
-;
-; #pragma clang loop vectorize(enable)
-; for (i1 = 0; i1 < 8; i1++) {
-; arr2[i1] = i1;
-; for (i2 = 0; i2 < 8; i2++)
-; arr[i2][i1] = i1 + n;
-; }
-; }
-;
-
- at arr2 = external global [8 x i32], align 16
- at arr = external global [8 x [8 x i32]], align 16
-
- at arrX = external global [8 x i64], align 16
- at arrY = external global [8 x [8 x i64]], align 16
-
-define void @foo_i32(i32 %n) {
-; CHECK-LABEL: define void @foo_i32(
-; CHECK-SAME: i32 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [8 x i32], ptr @arr2, i64 0, <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[TMP1:%.*]] = trunc <4 x i64> [[VEC_IND]] to <4 x i32>
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[TMP1]], <4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: br label %[[FOR_BODY31:.*]]
-; CHECK: [[FOR_BODY31]]:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[FOR_BODY31]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [8 x [8 x i32]], ptr @arr, i64 0, <4 x i64> [[VEC_PHI]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[TMP2]], <4 x ptr> align 4 [[TMP3]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP4]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], splat (i64 8)
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0
-; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[FOR_BODY31]]
-; CHECK: [[VECTOR_LATCH]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br i1 true, [[FOR_END10:label %.*]], label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-;
-entry:
- br label %for.body
-
-for.body: ; preds = %for.inc8, %entry
- %indvars.iv21 = phi i64 [ 0, %entry ], [ %indvars.iv.next22, %for.inc8 ]
- %arrayidx = getelementptr inbounds [8 x i32], ptr @arr2, i64 0, i64 %indvars.iv21
- %0 = trunc i64 %indvars.iv21 to i32
- store i32 %0, ptr %arrayidx, align 4
- %1 = trunc i64 %indvars.iv21 to i32
- %add = add nsw i32 %1, %n
- br label %for.body3
-
-for.body3: ; preds = %for.body3, %for.body
- %indvars.iv = phi i64 [ 0, %for.body ], [ %indvars.iv.next, %for.body3 ]
- %arrayidx7 = getelementptr inbounds [8 x [8 x i32]], ptr @arr, i64 0, i64 %indvars.iv, i64 %indvars.iv21
- store i32 %add, ptr %arrayidx7, align 4
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %exitcond = icmp eq i64 %indvars.iv.next, 8
- br i1 %exitcond, label %for.inc8, label %for.body3
-
-for.inc8: ; preds = %for.body3
- %indvars.iv.next22 = add nuw nsw i64 %indvars.iv21, 1
- %exitcond23 = icmp eq i64 %indvars.iv.next22, 8
- br i1 %exitcond23, label %for.end10, label %for.body, !llvm.loop !1
-
-for.end10: ; preds = %for.inc8
- ret void
-}
-
-define void @foo_i64(i64 %n) {
-; CHECK-LABEL: define void @foo_i64(
-; CHECK-SAME: i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[N]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [8 x i64], ptr @arrX, i64 0, <2 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> [[VEC_IND]], <2 x ptr> align 4 [[TMP0]], <2 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP1:%.*]] = add nsw <2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: br label %[[FOR_BODY31:.*]]
-; CHECK: [[FOR_BODY31]]:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[FOR_BODY31]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [8 x [8 x i64]], ptr @arrY, i64 0, <2 x i64> [[VEC_PHI]], <2 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v2i64.v2p0(<2 x i64> [[TMP1]], <2 x ptr> align 4 [[TMP2]], <2 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP3]] = add nuw nsw <2 x i64> [[VEC_PHI]], splat (i64 1)
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <2 x i64> [[TMP3]], splat (i64 8)
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
-; CHECK-NEXT: br i1 [[TMP5]], label %[[VECTOR_LATCH]], label %[[FOR_BODY31]]
-; CHECK: [[VECTOR_LATCH]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <2 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br i1 true, [[FOR_END10:label %.*]], label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-;
-entry:
- br label %for.body
-
-for.body: ; preds = %for.inc8, %entry
- %indvars.iv21 = phi i64 [ 0, %entry ], [ %indvars.iv.next22, %for.inc8 ]
- %arrayidx = getelementptr inbounds [8 x i64], ptr @arrX, i64 0, i64 %indvars.iv21
- store i64 %indvars.iv21, ptr %arrayidx, align 4
- %add = add nsw i64 %indvars.iv21, %n
- br label %for.body3
-
-for.body3: ; preds = %for.body3, %for.body
- %indvars.iv = phi i64 [ 0, %for.body ], [ %indvars.iv.next, %for.body3 ]
- %arrayidx7 = getelementptr inbounds [8 x [8 x i64]], ptr @arrY, i64 0, i64 %indvars.iv, i64 %indvars.iv21
- store i64 %add, ptr %arrayidx7, align 4
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %exitcond = icmp eq i64 %indvars.iv.next, 8
- br i1 %exitcond, label %for.inc8, label %for.body3
-
-for.inc8: ; preds = %for.body3
- %indvars.iv.next22 = add nuw nsw i64 %indvars.iv21, 1
- %exitcond23 = icmp eq i64 %indvars.iv.next22, 8
- br i1 %exitcond23, label %for.end10, label %for.body, !llvm.loop !1
-
-for.end10: ; preds = %for.inc8
- ret void
-}
-
-
-!1 = distinct !{!1, !2}
-!2 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-outer-loop.ll
deleted file mode 100644
index 6eff6d1df3523..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-printing-outer-loop.ll
+++ /dev/null
@@ -1,73 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
-
-; RUN: opt -passes=loop-vectorize -pass-remarks=loop-vectorize -enable-vplan-native-path -debug -disable-output %s 2>&1 | FileCheck %s
-
- at arr2 = external global [8 x i64], align 16
- at arr = external global [8 x [8 x i64]], align 16
-
-define void @foo(i64 %n) {
-; CHECK-LABEL: 'foo'
-; CHECK: VPlan 'Plain CFG
-; CHECK-NEXT: for UF>=1' {
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<entry>:
-; CHECK-NEXT: Successor(s): outer.header
-; CHECK-EMPTY:
-; CHECK-NEXT: outer.header:
-; CHECK-NEXT: EMIT-SCALAR ir<%outer.iv> = phi [ ir<%outer.iv.next>, outer.latch ], [ ir<0>, ir-bb<entry> ]
-; CHECK-NEXT: EMIT ir<%gep.1> = getelementptr inbounds ir<@arr2>, ir<0>, ir<%outer.iv>
-; CHECK-NEXT: EMIT store ir<%outer.iv>, ir<%gep.1>
-; CHECK-NEXT: EMIT ir<%add> = add nsw ir<%outer.iv>, ir<%n>
-; CHECK-NEXT: Successor(s): inner
-; CHECK-EMPTY:
-; CHECK-NEXT: inner:
-; CHECK-NEXT: EMIT-SCALAR ir<%inner.iv> = phi [ ir<%inner.iv.next>, inner ], [ ir<0>, outer.header ]
-; CHECK-NEXT: EMIT ir<%gep.2> = getelementptr inbounds ir<@arr>, ir<0>, ir<%inner.iv>, ir<%outer.iv>
-; CHECK-NEXT: EMIT store ir<%add>, ir<%gep.2>
-; CHECK-NEXT: EMIT ir<%inner.iv.next> = add nuw nsw ir<%inner.iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%inner.ec> = icmp eq ir<%inner.iv.next>, ir<8>
-; CHECK-NEXT: EMIT branch-on-cond ir<%inner.ec>
-; CHECK-NEXT: Successor(s): outer.latch, inner
-; CHECK-EMPTY:
-; CHECK-NEXT: outer.latch:
-; CHECK-NEXT: EMIT ir<%outer.iv.next> = add nuw nsw ir<%outer.iv>, ir<1>
-; CHECK-NEXT: EMIT ir<%outer.ec> = icmp eq ir<%outer.iv.next>, ir<8>
-; CHECK-NEXT: EMIT branch-on-cond ir<%outer.ec>
-; CHECK-NEXT: Successor(s): ir-bb<exit>, outer.header
-; CHECK-EMPTY:
-; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT: No successors
-; CHECK-NEXT: }
-;
-entry:
- br label %outer.header
-
-outer.header:
- %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
- %gep.1 = getelementptr inbounds [8 x i64], ptr @arr2, i64 0, i64 %outer.iv
- store i64 %outer.iv, ptr %gep.1, align 4
- %add = add nsw i64 %outer.iv, %n
- br label %inner
-
-inner:
- %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner ]
- %gep.2 = getelementptr inbounds [8 x [8 x i64]], ptr @arr, i64 0, i64 %inner.iv, i64 %outer.iv
- store i64 %add, ptr %gep.2, align 4
- %inner.iv.next = add nuw nsw i64 %inner.iv, 1
- %inner.ec = icmp eq i64 %inner.iv.next, 8
- br i1 %inner.ec, label %outer.latch, label %inner
-
-outer.latch:
- %outer.iv.next = add nuw nsw i64 %outer.iv, 1
- %outer.ec = icmp eq i64 %outer.iv.next, 8
- br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !1
-
-exit:
- ret void
-}
-
-; CHECK: remark: <unknown>:0:0: vectorized outer loop (vectorization width: 4, interleaved count: 1)
-
-!1 = distinct !{!1, !2, !3}
-!2 = !{!"llvm.loop.vectorize.width", i32 4}
-!3 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-stress-test-no-explict-vf.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-stress-test-no-explict-vf.ll
deleted file mode 100644
index 6c4262d98f3c8..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-stress-test-no-explict-vf.ll
+++ /dev/null
@@ -1,43 +0,0 @@
-; RUN: opt < %s -S -passes=loop-vectorize -enable-vplan-native-path -vplan-build-stress-test -debug-only=loop-vectorize -disable-output 2>&1 | FileCheck %s
-
-; This test checks that, when stress testing VPlan, if the computed VF
-; is 1, we override it to VF = 4.
-
-; CHECK: LV: VPlan computed VF 1.
-; CHECK: LV: VPlan stress testing: overriding computed VF.
-; CHECK: LV: Using VF 4 to build VPlans.
- at arr2 = external global [8 x i32], align 16
- at arr = external global [8 x [8 x i32]], align 16
-
-define void @foo(i32 %n) {
-entry:
- br label %for.body
-
-for.body: ; preds = %for.inc8, %entry
- %indvars.iv21 = phi i64 [ 0, %entry ], [ %indvars.iv.next22, %for.inc8 ]
- %arrayidx = getelementptr inbounds [8 x i32], ptr @arr2, i64 0, i64 %indvars.iv21
- %0 = trunc i64 %indvars.iv21 to i32
- store i32 %0, ptr %arrayidx, align 4
- %1 = trunc i64 %indvars.iv21 to i32
- %add = add nsw i32 %1, %n
- br label %for.body3
-
-for.body3: ; preds = %for.body3, %for.body
- %indvars.iv = phi i64 [ 0, %for.body ], [ %indvars.iv.next, %for.body3 ]
- %arrayidx7 = getelementptr inbounds [8 x [8 x i32]], ptr @arr, i64 0, i64 %indvars.iv, i64 %indvars.iv21
- store i32 %add, ptr %arrayidx7, align 4
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %exitcond = icmp eq i64 %indvars.iv.next, 8
- br i1 %exitcond, label %for.inc8, label %for.body3
-
-for.inc8: ; preds = %for.body3
- %indvars.iv.next22 = add nuw nsw i64 %indvars.iv21, 1
- %exitcond23 = icmp eq i64 %indvars.iv.next22, 8
- br i1 %exitcond23, label %for.end10, label %for.body, !llvm.loop !1
-
-for.end10: ; preds = %for.inc8
- ret void
-}
-
-!1 = distinct !{!1, !2}
-!2 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan_hcfg_stress_test.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan_hcfg_stress_test.ll
deleted file mode 100644
index f6b215f43d68e..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan_hcfg_stress_test.ll
+++ /dev/null
@@ -1,48 +0,0 @@
-; RUN: opt < %s -passes=loop-vectorize -enable-vplan-native-path -vplan-build-stress-test -debug-only=vplan -disable-output 2>&1 | FileCheck %s
-
-; Verify that the stress testing flag for the VPlan H-CFG builder works as
-; expected with and without enabling the VPlan H-CFG Verifier.
-
-; CHECK: VPlan 'Plain CFG
-
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-
-define void @foo(ptr nocapture %a, ptr nocapture readonly %b, i32 %N, i32 %M) {
-entry:
- %cmp32 = icmp sgt i32 %N, 0
- br i1 %cmp32, label %outer.ph, label %for.end15
-
-outer.ph:
- %cmp230 = icmp sgt i32 %M, 0
- %0 = sext i32 %M to i64
- %wide.trip.count = zext i32 %M to i64
- %wide.trip.count38 = zext i32 %N to i64
- br label %outer.body
-
-outer.body:
- %indvars.iv35 = phi i64 [ 0, %outer.ph ], [ %indvars.iv.next36, %outer.inc ]
- br i1 %cmp230, label %inner.ph, label %outer.inc
-
-inner.ph:
- %1 = mul nsw i64 %indvars.iv35, %0
- br label %inner.body
-
-inner.body:
- %indvars.iv = phi i64 [ 0, %inner.ph ], [ %indvars.iv.next, %inner.body ]
- %2 = add nsw i64 %indvars.iv, %1
- %arrayidx = getelementptr inbounds i32, ptr %b, i64 %2
- %3 = load i32, ptr %arrayidx, align 4
- %arrayidx12 = getelementptr inbounds i32, ptr %a, i64 %2
- store i32 %3, ptr %arrayidx12, align 4
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
- br i1 %exitcond, label %outer.inc, label %inner.body
-
-outer.inc:
- %indvars.iv.next36 = add nuw nsw i64 %indvars.iv35, 1
- %exitcond39 = icmp eq i64 %indvars.iv.next36, %wide.trip.count38
- br i1 %exitcond39, label %for.end15, label %outer.body
-
-for.end15:
- ret void
-}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll b/llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll
deleted file mode 100644
index 527709c458c62..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll
+++ /dev/null
@@ -1,115 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 5
-; RUN: opt -S -passes=loop-vectorize -enable-vplan-native-path -mtriple x86_64 < %s | FileCheck %s
-; RUN: opt -S -passes=loop-vectorize -enable-vplan-native-path -mtriple x86_64 -mattr=+avx < %s | FileCheck %s --check-prefix=AVX
-; RUN: opt -S -passes=loop-vectorize -enable-vplan-native-path -mtriple x86_64 -mattr=+avx2 < %s | FileCheck %s --check-prefix=AVX
-
-; extern int arr[8][8];
-; extern int arr2[8];
-;
-; void foo(int n)
-; {
-; int i1, i2;
-;
-; #pragma clang loop vectorize(enable)
-; for (i1 = 0; i1 < 8; i1++) {
-; arr2[i1] = i1;
-; for (i2 = 0; i2 < 8; i2++)
-; arr[i2][i1] = i1 + n;
-; }
-; }
-;
-
- at arr2 = external global [8 x i32], align 16
- at arr = external global [8 x [8 x i32]], align 16
-
-define void @foo(i32 %n) {
-; CHECK-LABEL: define void @foo(
-; CHECK-SAME: i32 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [8 x i32], ptr @arr2, i64 0, <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[TMP1:%.*]] = trunc <4 x i64> [[VEC_IND]] to <4 x i32>
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[TMP1]], <4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: br label %[[FOR_BODY31:.*]]
-; CHECK: [[FOR_BODY31]]:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[FOR_BODY31]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [8 x [8 x i32]], ptr @arr, i64 0, <4 x i64> [[VEC_PHI]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[TMP2]], <4 x ptr> align 4 [[TMP3]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP4]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], splat (i64 8)
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0
-; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[FOR_BODY31]]
-; CHECK: [[VECTOR_LATCH]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br i1 true, [[FOR_END10:label %.*]], label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-;
-; AVX-LABEL: define void @foo(
-; AVX-SAME: i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; AVX-NEXT: [[ENTRY:.*:]]
-; AVX-NEXT: br label %[[VECTOR_PH:.*]]
-; AVX: [[VECTOR_PH]]:
-; AVX-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[N]], i64 0
-; AVX-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
-; AVX-NEXT: br label %[[VECTOR_BODY:.*]]
-; AVX: [[VECTOR_BODY]]:
-; AVX-NEXT: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, <8 x ptr> align 4 getelementptr inbounds ([8 x i32], ptr @arr2, <8 x i64> zeroinitializer, <8 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>), <8 x i1> splat (i1 true))
-; AVX-NEXT: [[TMP2:%.*]] = add nsw <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[BROADCAST_SPLAT]]
-; AVX-NEXT: br label %[[FOR_BODY31:.*]]
-; AVX: [[FOR_BODY31]]:
-; AVX-NEXT: [[VEC_PHI:%.*]] = phi <8 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[FOR_BODY31]] ]
-; AVX-NEXT: [[TMP3:%.*]] = getelementptr inbounds [8 x [8 x i32]], ptr @arr, i64 0, <8 x i64> [[VEC_PHI]], <8 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
-; AVX-NEXT: call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> [[TMP2]], <8 x ptr> align 4 [[TMP3]], <8 x i1> splat (i1 true))
-; AVX-NEXT: [[TMP4]] = add nuw nsw <8 x i64> [[VEC_PHI]], splat (i64 1)
-; AVX-NEXT: [[TMP5:%.*]] = icmp eq <8 x i64> [[TMP4]], splat (i64 8)
-; AVX-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i32 0
-; AVX-NEXT: br i1 [[TMP6]], label %[[VECTOR_LATCH:.*]], label %[[FOR_BODY31]], !llvm.loop [[LOOP0:![0-9]+]]
-; AVX: [[VECTOR_LATCH]]:
-; AVX-NEXT: br label %[[MIDDLE_BLOCK:.*]]
-; AVX: [[MIDDLE_BLOCK]]:
-; AVX-NEXT: br i1 true, [[FOR_END10:label %.*]], label %[[SCALAR_PH:.*]]
-; AVX: [[SCALAR_PH]]:
-;
-entry:
- br label %for.body
-
-for.body: ; preds = %for.inc8, %entry
- %indvars.iv21 = phi i64 [ 0, %entry ], [ %indvars.iv.next22, %for.inc8 ]
- %arrayidx = getelementptr inbounds [8 x i32], ptr @arr2, i64 0, i64 %indvars.iv21
- %0 = trunc i64 %indvars.iv21 to i32
- store i32 %0, ptr %arrayidx, align 4
- %1 = trunc i64 %indvars.iv21 to i32
- %add = add nsw i32 %1, %n
- br label %for.body3
-
-for.body3: ; preds = %for.body3, %for.body
- %indvars.iv = phi i64 [ 0, %for.body ], [ %indvars.iv.next, %for.body3 ]
- %arrayidx7 = getelementptr inbounds [8 x [8 x i32]], ptr @arr, i64 0, i64 %indvars.iv, i64 %indvars.iv21
- store i32 %add, ptr %arrayidx7, align 4
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %exitcond = icmp eq i64 %indvars.iv.next, 8
- br i1 %exitcond, label %for.inc8, label %for.body3
-
-for.inc8: ; preds = %for.body3
- %indvars.iv.next22 = add nuw nsw i64 %indvars.iv21, 1
- %exitcond23 = icmp eq i64 %indvars.iv.next22, 8
- br i1 %exitcond23, label %for.end10, label %for.body, !llvm.loop !1
-
-for.end10: ; preds = %for.inc8
- ret void
-}
-
-!1 = distinct !{!1, !2}
-!2 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vplan-native-inner-loop-only.ll b/llvm/test/Transforms/LoopVectorize/X86/vplan-native-inner-loop-only.ll
deleted file mode 100644
index d4335961e19b2..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/X86/vplan-native-inner-loop-only.ll
+++ /dev/null
@@ -1,83 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -p loop-vectorize -enable-vplan-native-path -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
-; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
-
-target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-; -enable-vplan-native-path should not impact codegen for inner loops.
-
-define void @test(ptr %A) {
-; CHECK-LABEL: define void @test(
-; CHECK-SAME: ptr [[A:%.*]]) {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT: [[TMP4:%.*]] = shl nsw i64 [[INDEX]], 1
-; CHECK-NEXT: [[TMP5:%.*]] = shl nsw i64 [[TMP1]], 1
-; CHECK-NEXT: [[TMP6:%.*]] = shl nsw i64 [[TMP2]], 1
-; CHECK-NEXT: [[TMP7:%.*]] = shl nsw i64 [[TMP3]], 1
-; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP7]]
-; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP8]], align 4
-; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT: [[TMP13:%.*]] = add <4 x i32> [[STRIDED_VEC]], splat (i32 2)
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i32> [[TMP13]], i32 0
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i32> [[TMP13]], i32 1
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i32> [[TMP13]], i32 2
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i32> [[TMP13]], i32 3
-; CHECK-NEXT: store i32 [[TMP14]], ptr [[TMP8]], align 4
-; CHECK-NEXT: store i32 [[TMP15]], ptr [[TMP9]], align 4
-; CHECK-NEXT: store i32 [[TMP16]], ptr [[TMP10]], align 4
-; CHECK-NEXT: store i32 [[TMP17]], ptr [[TMP11]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: br label [[SCALAR_PH:%.*]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: br label [[LOOP:%.*]]
-; CHECK: loop:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 96, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT: [[MUL:%.*]] = shl nsw i64 [[IV]], 1
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[MUL]]
-; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
-; CHECK-NEXT: [[ADD:%.*]] = add i32 [[L]], 2
-; CHECK-NEXT: store i32 [[ADD]], ptr [[GEP]], align 4
-; CHECK-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 100
-; CHECK-NEXT: br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: exit:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop
-
-loop:
- %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
- %mul = shl nsw i64 %iv, 1
- %gep = getelementptr inbounds i32, ptr %A, i64 %mul
- %l = load i32, ptr %gep, align 4
- %add = add i32 %l, 2
- store i32 %add, ptr %gep
- %iv.next = add nsw i64 %iv, 1
- %ec = icmp eq i64 %iv.next, 100
- br i1 %ec, label %exit, label %loop
-
-exit:
- ret void
-}
-;.
-; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
-;.
diff --git a/llvm/test/Transforms/LoopVectorize/dbg-outer-loop-vect.ll b/llvm/test/Transforms/LoopVectorize/dbg-outer-loop-vect.ll
deleted file mode 100644
index 9e2e74d14c95a..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/dbg-outer-loop-vect.ll
+++ /dev/null
@@ -1,172 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -S -passes=loop-vectorize -enable-vplan-native-path -force-vector-interleave=1 -force-vector-width=4 | FileCheck %s
-
-target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
-
-define void @foo(ptr %h) !dbg !4 {
-; CHECK-LABEL: define void @foo(
-; CHECK-SAME: ptr [[H:%.*]]) !dbg [[DBG4:![0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: #dbg_value(i64 0, [[META11:![0-9]+]], !DIExpression(), [[META20:![0-9]+]])
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]], !dbg [[DBG21:![0-9]+]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]], !dbg [[DBG21]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ], !dbg [[DBG22:![0-9]+]]
-; CHECK-NEXT: br label %[[FOR_COND5_PREHEADER1:.*]], !dbg [[DBG21]]
-; CHECK: [[FOR_COND5_PREHEADER1]]:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP5:%.*]], %[[FOR_COND5_PREHEADER1]] ], !dbg [[DBG23:![0-9]+]]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[H]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> zeroinitializer, <4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true)), !dbg [[DBG24:![0-9]+]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, <4 x ptr> [[TMP0]], i64 1, !dbg [[DBG26:![0-9]+]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> splat (i32 1), <4 x ptr> align 4 [[TMP2]], <4 x i1> splat (i1 true)), !dbg [[DBG24]]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, <4 x ptr> [[TMP0]], i64 2, !dbg [[DBG26]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> splat (i32 2), <4 x ptr> align 4 [[TMP3]], <4 x i1> splat (i1 true)), !dbg [[DBG24]]
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, <4 x ptr> [[TMP0]], i64 3, !dbg [[DBG26]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> splat (i32 3), <4 x ptr> align 4 [[TMP4]], <4 x i1> splat (i1 true)), !dbg [[DBG24]]
-; CHECK-NEXT: [[TMP5]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1), !dbg [[DBG27:![0-9]+]]
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP5]], splat (i64 5), !dbg [[DBG28:![0-9]+]]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0, !dbg [[DBG29:![0-9]+]]
-; CHECK-NEXT: br i1 [[TMP7]], label %[[VECTOR_LATCH]], label %[[FOR_COND5_PREHEADER1]], !dbg [[DBG29]]
-; CHECK: [[VECTOR_LATCH]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4, !dbg [[DBG22]]
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20, !dbg [[DBG21]]
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !dbg [[DBG21]], !llvm.loop [[LOOP30:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH:.*]], !dbg [[DBG21]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[FOR_COND1_PREHEADER:.*]], !dbg [[DBG21]]
-; CHECK: [[FOR_COND1_PREHEADER]]:
-; CHECK-NEXT: [[I_023:%.*]] = phi i64 [ 20, %[[SCALAR_PH]] ], [ [[INC13:%.*]], %[[FOR_COND_CLEANUP3:.*]] ]
-; CHECK-NEXT: #dbg_value(i64 [[I_023]], [[META11]], !DIExpression(), [[META20]])
-; CHECK-NEXT: br label %[[FOR_COND5_PREHEADER:.*]], !dbg [[DBG29]]
-; CHECK: [[FOR_COND5_PREHEADER]]:
-; CHECK-NEXT: [[L_022:%.*]] = phi i64 [ 0, %[[FOR_COND1_PREHEADER]] ], [ [[INC10:%.*]], %[[FOR_COND5_PREHEADER]] ], !dbg [[DBG23]]
-; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i32, ptr [[H]], i64 [[L_022]]
-; CHECK-NEXT: store i32 0, ptr [[TMP9]], align 4, !dbg [[DBG24]]
-; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr i32, ptr [[TMP9]], i64 1, !dbg [[DBG26]]
-; CHECK-NEXT: store i32 1, ptr [[ARRAYIDX_1]], align 4, !dbg [[DBG24]]
-; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr i32, ptr [[TMP9]], i64 2, !dbg [[DBG26]]
-; CHECK-NEXT: store i32 2, ptr [[ARRAYIDX_2]], align 4, !dbg [[DBG24]]
-; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr i32, ptr [[TMP9]], i64 3, !dbg [[DBG26]]
-; CHECK-NEXT: store i32 3, ptr [[ARRAYIDX_3]], align 4, !dbg [[DBG24]]
-; CHECK-NEXT: [[INC10]] = add nuw nsw i64 [[L_022]], 1, !dbg [[DBG27]]
-; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC10]], 5, !dbg [[DBG28]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP3]], label %[[FOR_COND5_PREHEADER]], !dbg [[DBG29]]
-; CHECK: [[FOR_COND_CLEANUP3]]:
-; CHECK-NEXT: [[INC13]] = add nuw nsw i64 [[I_023]], 1, !dbg [[DBG22]]
-; CHECK-NEXT: #dbg_value(i64 [[INC13]], [[META11]], !DIExpression(), [[META20]])
-; CHECK-NEXT: [[EXITCOND24_NOT:%.*]] = icmp eq i64 [[INC13]], 23, !dbg [[DBG33:![0-9]+]]
-; CHECK-NEXT: br i1 [[EXITCOND24_NOT]], label %[[EXIT]], label %[[FOR_COND1_PREHEADER]], !dbg [[DBG21]], !llvm.loop [[LOOP34:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void, !dbg [[DBG35:![0-9]+]]
-;
-entry:
- call void @llvm.dbg.value(metadata i64 0, metadata !11, metadata !DIExpression()), !dbg !20
- br label %for.cond1.preheader, !dbg !21
-
-for.cond1.preheader: ; preds = %entry, %for.cond.cleanup3
- %i.023 = phi i64 [ 0, %entry ], [ %inc13, %for.cond.cleanup3 ]
- call void @llvm.dbg.value(metadata i64 %i.023, metadata !11, metadata !DIExpression()), !dbg !20
- br label %for.cond5.preheader, !dbg !22
-
-for.cond5.preheader: ; preds = %for.cond1.preheader, %for.cond5.preheader
- %l.022 = phi i64 [ 0, %for.cond1.preheader ], [ %inc10, %for.cond5.preheader ], !dbg !34
- %0 = getelementptr i32, ptr %h, i64 %l.022
- store i32 0, ptr %0, align 4, !dbg !24
- %arrayidx.1 = getelementptr i32, ptr %0, i64 1, !dbg !26
- store i32 1, ptr %arrayidx.1, align 4, !dbg !24
- %arrayidx.2 = getelementptr i32, ptr %0, i64 2, !dbg !26
- store i32 2, ptr %arrayidx.2, align 4, !dbg !24
- %arrayidx.3 = getelementptr i32, ptr %0, i64 3, !dbg !26
- store i32 3, ptr %arrayidx.3, align 4, !dbg !24
- %inc10 = add nuw nsw i64 %l.022, 1, !dbg !27
- %exitcond.not = icmp eq i64 %inc10, 5, !dbg !28
- br i1 %exitcond.not, label %for.cond.cleanup3, label %for.cond5.preheader, !dbg !22
-
-for.cond.cleanup3: ; preds = %for.cond5.preheader
- %inc13 = add nuw nsw i64 %i.023, 1, !dbg !29
- call void @llvm.dbg.value(metadata i64 %inc13, metadata !11, metadata !DIExpression()), !dbg !20
- %exitcond24.not = icmp eq i64 %inc13, 23, !dbg !30
- br i1 %exitcond24.not, label %exit, label %for.cond1.preheader, !dbg !21, !llvm.loop !31
-
-exit: ; preds = %for.cond.cleanup3
- ret void, !dbg !23
-}
-
-declare void @llvm.dbg.value(metadata, metadata, metadata)
-
-!llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!2, !3}
-
-!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
-!1 = !DIFile(filename: "outer-loop-vect.c", directory: "/test/file/path")
-!2 = !{i32 2, !"Debug Info Version", i32 3}
-!3 = !{i32 7, !"debug-info-assignment-tracking", i1 true}
-!4 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 8, type: !5, scopeLine: 8, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !9)
-!5 = !DISubroutineType(types: !6)
-!6 = !{null, !7}
-!7 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !8, size: 64)
-!8 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!9 = !{!10, !11, !14, !17}
-!10 = !DILocalVariable(name: "h", arg: 1, scope: !4, file: !1, line: 8, type: !7)
-!11 = !DILocalVariable(name: "i", scope: !12, file: !1, line: 10, type: !13)
-!12 = distinct !DILexicalBlock(scope: !4, file: !1, line: 10, column: 3)
-!13 = !DIBasicType(name: "unsigned long", size: 64, encoding: DW_ATE_unsigned)
-!14 = !DILocalVariable(name: "l", scope: !15, file: !1, line: 11, type: !13)
-!15 = distinct !DILexicalBlock(scope: !16, file: !1, line: 11, column: 5)
-!16 = distinct !DILexicalBlock(scope: !12, file: !1, line: 10, column: 3)
-!17 = !DILocalVariable(name: "j", scope: !18, file: !1, line: 12, type: !13)
-!18 = distinct !DILexicalBlock(scope: !19, file: !1, line: 12, column: 7)
-!19 = distinct !DILexicalBlock(scope: !15, file: !1, line: 11, column: 5)
-!20 = !DILocation(line: 0, scope: !12)
-!21 = !DILocation(line: 10, column: 3, scope: !12)
-!22 = !DILocation(line: 11, column: 5, scope: !15)
-!23 = !DILocation(line: 14, column: 1, scope: !4)
-!24 = !DILocation(line: 13, column: 11, scope: !25)
-!25 = distinct !DILexicalBlock(scope: !18, file: !1, line: 12, column: 7)
-!26 = !DILocation(line: 13, column: 2, scope: !25)
-!27 = !DILocation(line: 11, column: 32, scope: !19)
-!28 = !DILocation(line: 11, column: 26, scope: !19)
-!29 = !DILocation(line: 10, column: 30, scope: !16)
-!30 = !DILocation(line: 10, column: 24, scope: !16)
-!31 = distinct !{!31, !21, !32, !33}
-!32 = !DILocation(line: 13, column: 13, scope: !12)
-!33 = !{!"llvm.loop.vectorize.enable", i1 true}
-!34 = !DILocation(line: 10, column: 5, scope: !12)
-;.
-; CHECK: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C11, file: [[META1:![0-9]+]], producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
-; CHECK: [[META1]] = !DIFile(filename: "{{.*}}outer-loop-vect.c", directory: {{.*}})
-; CHECK: [[DBG4]] = distinct !DISubprogram(name: "foo", scope: [[META1]], file: [[META1]], line: 8, type: [[META5:![0-9]+]], scopeLine: 8, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META9:![0-9]+]])
-; CHECK: [[META5]] = !DISubroutineType(types: [[META6:![0-9]+]])
-; CHECK: [[META6]] = !{null, [[META7:![0-9]+]]}
-; CHECK: [[META7]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: [[META8:![0-9]+]], size: 64)
-; CHECK: [[META8]] = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-; CHECK: [[META9]] = !{[[META10:![0-9]+]], [[META11]], [[META14:![0-9]+]], [[META17:![0-9]+]]}
-; CHECK: [[META10]] = !DILocalVariable(name: "h", arg: 1, scope: [[DBG4]], file: [[META1]], line: 8, type: [[META7]])
-; CHECK: [[META11]] = !DILocalVariable(name: "i", scope: [[META12:![0-9]+]], file: [[META1]], line: 10, type: [[META13:![0-9]+]])
-; CHECK: [[META12]] = distinct !DILexicalBlock(scope: [[DBG4]], file: [[META1]], line: 10, column: 3)
-; CHECK: [[META13]] = !DIBasicType(name: "unsigned long", size: 64, encoding: DW_ATE_unsigned)
-; CHECK: [[META14]] = !DILocalVariable(name: "l", scope: [[META15:![0-9]+]], file: [[META1]], line: 11, type: [[META13]])
-; CHECK: [[META15]] = distinct !DILexicalBlock(scope: [[META16:![0-9]+]], file: [[META1]], line: 11, column: 5)
-; CHECK: [[META16]] = distinct !DILexicalBlock(scope: [[META12]], file: [[META1]], line: 10, column: 3)
-; CHECK: [[META17]] = !DILocalVariable(name: "j", scope: [[META18:![0-9]+]], file: [[META1]], line: 12, type: [[META13]])
-; CHECK: [[META18]] = distinct !DILexicalBlock(scope: [[META19:![0-9]+]], file: [[META1]], line: 12, column: 7)
-; CHECK: [[META19]] = distinct !DILexicalBlock(scope: [[META15]], file: [[META1]], line: 11, column: 5)
-; CHECK: [[META20]] = !DILocation(line: 0, scope: [[META12]])
-; CHECK: [[DBG21]] = !DILocation(line: 10, column: 3, scope: [[META12]])
-; CHECK: [[DBG22]] = !DILocation(line: 10, column: 30, scope: [[META16]])
-; CHECK: [[DBG23]] = !DILocation(line: 10, column: 5, scope: [[META12]])
-; CHECK: [[DBG24]] = !DILocation(line: 13, column: 11, scope: [[META25:![0-9]+]])
-; CHECK: [[META25]] = distinct !DILexicalBlock(scope: [[META18]], file: [[META1]], line: 12, column: 7)
-; CHECK: [[DBG26]] = !DILocation(line: 13, column: 2, scope: [[META25]])
-; CHECK: [[DBG27]] = !DILocation(line: 11, column: 32, scope: [[META19]])
-; CHECK: [[DBG28]] = !DILocation(line: 11, column: 26, scope: [[META19]])
-; CHECK: [[DBG29]] = !DILocation(line: 11, column: 5, scope: [[META15]])
-; CHECK: [[LOOP30]] = distinct !{[[LOOP30]], [[META31:![0-9]+]], [[META32:![0-9]+]]}
-; CHECK: [[META31]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META32]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[DBG33]] = !DILocation(line: 10, column: 24, scope: [[META16]])
-; CHECK: [[LOOP34]] = distinct !{[[LOOP34]], [[META32]], [[META31]]}
-; CHECK: [[DBG35]] = !DILocation(line: 14, column: 1, scope: [[DBG4]])
-;.
diff --git a/llvm/test/Transforms/LoopVectorize/explicit_outer_detection.ll b/llvm/test/Transforms/LoopVectorize/explicit_outer_detection.ll
deleted file mode 100644
index f780dd3fa7cf8..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/explicit_outer_detection.ll
+++ /dev/null
@@ -1,236 +0,0 @@
-; RUN: opt < %s -passes=loop-vectorize -enable-vplan-native-path -debug-only=loop-vectorize --disable-output -S 2>&1 | FileCheck %s
-; REQUIRES: asserts
-
-; Verify that outer loops annotated only with the expected explicit
-; vectorization hints are collected for vectorization instead of inner loops.
-
-; Root C/C++ source code for all the test cases
-; void foo(int *a, int *b, int N, int M)
-; {
-; int i, j;
-; #pragma clang loop vectorize(enable)
-; for (i = 0; i < N; i++) {
-; for (j = 0; j < M; j++) {
-; a[i*M+j] = bptr b[i*M+j];
-; }
-; }
-; }
-
-; Case 1: Annotated outer loop WITH vector width information must be collected.
-
-; CHECK-LABEL: vector_width
-; CHECK: LV: Loop hints: force=enabled width=4 interleave=0
-; CHECK: LV: We can vectorize this outer loop!
-; CHECK: LV: Using user VF 4 to build VPlans.
-; CHECK-NOT: LV: Loop hints: force=?
-; CHECK-NOT: LV: Found a loop: inner.body
-
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-
-define void @vector_width(ptr nocapture %a, ptr nocapture readonly %b, i32 %N, i32 %M) {
-entry:
- %cmp32 = icmp sgt i32 %N, 0
- br i1 %cmp32, label %outer.ph, label %for.end15
-
-outer.ph: ; preds = %entry
- %cmp230 = icmp sgt i32 %M, 0
- %0 = sext i32 %M to i64
- %wide.trip.count = zext i32 %M to i64
- %wide.trip.count38 = zext i32 %N to i64
- br label %outer.body
-
-outer.body: ; preds = %outer.inc, %outer.ph
- %indvars.iv35 = phi i64 [ 0, %outer.ph ], [ %indvars.iv.next36, %outer.inc ]
- br i1 %cmp230, label %inner.ph, label %outer.inc
-
-inner.ph: ; preds = %outer.body
- %1 = mul nsw i64 %indvars.iv35, %0
- br label %inner.body
-
-inner.body: ; preds = %inner.body, %inner.ph
- %indvars.iv = phi i64 [ 0, %inner.ph ], [ %indvars.iv.next, %inner.body ]
- %2 = add nsw i64 %indvars.iv, %1
- %arrayidx = getelementptr inbounds i32, ptr %b, i64 %2
- %3 = load i32, ptr %arrayidx, align 4, !tbaa !2
- %mul8 = mul nsw i32 %3, %3
- %arrayidx12 = getelementptr inbounds i32, ptr %a, i64 %2
- store i32 %mul8, ptr %arrayidx12, align 4, !tbaa !2
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
- br i1 %exitcond, label %outer.inc, label %inner.body
-
-outer.inc: ; preds = %inner.body, %outer.body
- %indvars.iv.next36 = add nuw nsw i64 %indvars.iv35, 1
- %exitcond39 = icmp eq i64 %indvars.iv.next36, %wide.trip.count38
- br i1 %exitcond39, label %for.end15, label %outer.body, !llvm.loop !6
-
-for.end15: ; preds = %outer.inc, %entry
- ret void
-}
-
-; Case 2: Annotated outer loop WITHOUT vector width information must be collected.
-
-; CHECK-LABEL: case2
-; CHECK: LV: Loop hints: force=enabled width=0 interleave=0
-; CHECK: LV: We can vectorize this outer loop!
-; CHECK: LV: Using VF 1 to build VPlans.
-
-define void @case2(ptr nocapture %a, ptr nocapture readonly %b, i32 %N, i32 %M) {
-entry:
- %cmp32 = icmp sgt i32 %N, 0
- br i1 %cmp32, label %outer.ph, label %for.end15
-
-outer.ph: ; preds = %entry
- %cmp230 = icmp sgt i32 %M, 0
- %0 = sext i32 %M to i64
- %wide.trip.count = zext i32 %M to i64
- %wide.trip.count38 = zext i32 %N to i64
- br label %outer.body
-
-outer.body: ; preds = %outer.inc, %outer.ph
- %indvars.iv35 = phi i64 [ 0, %outer.ph ], [ %indvars.iv.next36, %outer.inc ]
- br i1 %cmp230, label %inner.ph, label %outer.inc
-
-inner.ph: ; preds = %outer.body
- %1 = mul nsw i64 %indvars.iv35, %0
- br label %inner.body
-
-inner.body: ; preds = %inner.body, %inner.ph
- %indvars.iv = phi i64 [ 0, %inner.ph ], [ %indvars.iv.next, %inner.body ]
- %2 = add nsw i64 %indvars.iv, %1
- %arrayidx = getelementptr inbounds i32, ptr %b, i64 %2
- %3 = load i32, ptr %arrayidx, align 4, !tbaa !2
- %mul8 = mul nsw i32 %3, %3
- %arrayidx12 = getelementptr inbounds i32, ptr %a, i64 %2
- store i32 %mul8, ptr %arrayidx12, align 4, !tbaa !2
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
- br i1 %exitcond, label %outer.inc, label %inner.body
-
-outer.inc: ; preds = %inner.body, %outer.body
- %indvars.iv.next36 = add nuw nsw i64 %indvars.iv35, 1
- %exitcond39 = icmp eq i64 %indvars.iv.next36, %wide.trip.count38
- br i1 %exitcond39, label %for.end15, label %outer.body, !llvm.loop !9
-
-for.end15: ; preds = %outer.inc, %entry
- ret void
-}
-
-; Case 3: Annotated outer loop WITH vector width and interleave information
-; doesn't have to be collected.
-
-; CHECK-LABEL: case3
-; CHECK-NOT: LV: Loop hints: force=enabled
-; CHECK-NOT: LV: We can vectorize this outer loop!
-; CHECK: LV: Loop hints: force=?
-; CHECK: LV: Found a loop: inner.body
-
-define void @case3(ptr nocapture %a, ptr nocapture readonly %b, i32 %N, i32 %M) {
-entry:
- %cmp32 = icmp sgt i32 %N, 0
- br i1 %cmp32, label %outer.ph, label %for.end15
-
-outer.ph: ; preds = %entry
- %cmp230 = icmp sgt i32 %M, 0
- %0 = sext i32 %M to i64
- %wide.trip.count = zext i32 %M to i64
- %wide.trip.count38 = zext i32 %N to i64
- br label %outer.body
-
-outer.body: ; preds = %outer.inc, %outer.ph
- %indvars.iv35 = phi i64 [ 0, %outer.ph ], [ %indvars.iv.next36, %outer.inc ]
- br i1 %cmp230, label %inner.ph, label %outer.inc
-
-inner.ph: ; preds = %outer.body
- %1 = mul nsw i64 %indvars.iv35, %0
- br label %inner.body
-
-inner.body: ; preds = %inner.body, %inner.ph
- %indvars.iv = phi i64 [ 0, %inner.ph ], [ %indvars.iv.next, %inner.body ]
- %2 = add nsw i64 %indvars.iv, %1
- %arrayidx = getelementptr inbounds i32, ptr %b, i64 %2
- %3 = load i32, ptr %arrayidx, align 4, !tbaa !2
- %mul8 = mul nsw i32 %3, %3
- %arrayidx12 = getelementptr inbounds i32, ptr %a, i64 %2
- store i32 %mul8, ptr %arrayidx12, align 4, !tbaa !2
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
- br i1 %exitcond, label %outer.inc, label %inner.body
-
-outer.inc: ; preds = %inner.body, %outer.body
- %indvars.iv.next36 = add nuw nsw i64 %indvars.iv35, 1
- %exitcond39 = icmp eq i64 %indvars.iv.next36, %wide.trip.count38
- br i1 %exitcond39, label %for.end15, label %outer.body, !llvm.loop !11
-
-for.end15: ; preds = %outer.inc, %entry
- ret void
-}
-
-; Case 4: Outer loop without any explicit vectorization annotation doesn't have
-; to be collected.
-
-; CHECK-LABEL: case4
-; CHECK-NOT: LV: Loop hints: force=enabled
-; CHECK-NOT: LV: We can vectorize this outer loop!
-; CHECK: LV: Loop hints: force=?
-; CHECK: LV: Found a loop: inner.body
-
-define void @case4(ptr nocapture %a, ptr nocapture readonly %b, i32 %N, i32 %M) {
-entry:
- %cmp32 = icmp sgt i32 %N, 0
- br i1 %cmp32, label %outer.ph, label %for.end15
-
-outer.ph: ; preds = %entry
- %cmp230 = icmp sgt i32 %M, 0
- %0 = sext i32 %M to i64
- %wide.trip.count = zext i32 %M to i64
- %wide.trip.count38 = zext i32 %N to i64
- br label %outer.body
-
-outer.body: ; preds = %outer.inc, %outer.ph
- %indvars.iv35 = phi i64 [ 0, %outer.ph ], [ %indvars.iv.next36, %outer.inc ]
- br i1 %cmp230, label %inner.ph, label %outer.inc
-
-inner.ph: ; preds = %outer.body
- %1 = mul nsw i64 %indvars.iv35, %0
- br label %inner.body
-
-inner.body: ; preds = %inner.body, %inner.ph
- %indvars.iv = phi i64 [ 0, %inner.ph ], [ %indvars.iv.next, %inner.body ]
- %2 = add nsw i64 %indvars.iv, %1
- %arrayidx = getelementptr inbounds i32, ptr %b, i64 %2
- %3 = load i32, ptr %arrayidx, align 4, !tbaa !2
- %mul8 = mul nsw i32 %3, %3
- %arrayidx12 = getelementptr inbounds i32, ptr %a, i64 %2
- store i32 %mul8, ptr %arrayidx12, align 4, !tbaa !2
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
- br i1 %exitcond, label %outer.inc, label %inner.body
-
-outer.inc: ; preds = %inner.body, %outer.body
- %indvars.iv.next36 = add nuw nsw i64 %indvars.iv35, 1
- %exitcond39 = icmp eq i64 %indvars.iv.next36, %wide.trip.count38
- br i1 %exitcond39, label %for.end15, label %outer.body
-
-for.end15: ; preds = %outer.inc, %entry
- ret void
-}
-
-!llvm.module.flags = !{!0}
-!llvm.ident = !{!1}
-
-!0 = !{i32 1, !"wchar_size", i32 4}
-!1 = !{!"clang version 6.0.0"}
-!2 = !{!3, !3, i64 0}
-!3 = !{!"int", !4, i64 0}
-!4 = !{!"omnipotent char", !5, i64 0}
-!5 = !{!"Simple C/C++ TBAA"}
-; Case 1
-!6 = distinct !{!6, !7, !8}
-!7 = !{!"llvm.loop.vectorize.width", i32 4}
-!8 = !{!"llvm.loop.vectorize.enable", i1 true}
-; Case 2
-!9 = distinct !{!9, !8}
-; Case 3
-!10 = !{!"llvm.loop.interleave.count", i32 2}
-!11 = distinct !{!11, !7, !10, !8}
diff --git a/llvm/test/Transforms/LoopVectorize/explicit_outer_nonuniform_inner.ll b/llvm/test/Transforms/LoopVectorize/explicit_outer_nonuniform_inner.ll
deleted file mode 100644
index e3fb756d056b5..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/explicit_outer_nonuniform_inner.ll
+++ /dev/null
@@ -1,173 +0,0 @@
-; RUN: opt < %s -passes=loop-vectorize -enable-vplan-native-path -pass-remarks-analysis=loop-vectorize -debug-only=loop-vectorize --disable-output -S 2>&1 | FileCheck %s
-; REQUIRES: asserts
-
-; Verify that LV bails out on explicit vectorization outer loops that contain
-; divergent inner loops.
-
-; Root C/C++ source code for all the test cases
-; void foo(int *a, int *b, int N, int M)
-; {
-; int i, j;
-; #pragma clang loop vectorize(enable) vectorize_width(8)
-; for (i = 0; i < N; i++) {
-; // Tested inner loop. It will be replaced per test.
-; for (j = 0; j < M; j++) {
-; a[i*M+j] = bptr b[i*M+j];
-; }
-; }
-; }
-
-; Case 1 (for (j = i; j < M; j++)): Inner loop with divergent IV start.
-
-; CHECK-LABEL: iv_start
-; CHECK: LV: Not vectorizing: Outer loop contains divergent loops.
-; CHECK: LV: Not vectorizing: Unsupported outer loop.
-
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-
-define void @iv_start(ptr nocapture %a, ptr nocapture readonly %b, i32 %N, i32 %M) {
-entry:
- %cmp33 = icmp sgt i32 %N, 0
- br i1 %cmp33, label %outer.ph, label %for.end15
-
-outer.ph: ; preds = %entry
- %0 = sext i32 %M to i64
- %wide.trip.count = zext i32 %M to i64
- %wide.trip.count41 = zext i32 %N to i64
- br label %outer.body
-
-outer.body: ; preds = %outer.inc, %outer.ph
- %indvars.iv38 = phi i64 [ 0, %outer.ph ], [ %indvars.iv.next39, %outer.inc ]
- %cmp231 = icmp slt i64 %indvars.iv38, %0
- br i1 %cmp231, label %inner.ph, label %outer.inc
-
-inner.ph: ; preds = %outer.body
- %1 = mul nsw i64 %indvars.iv38, %0
- br label %inner.body
-
-inner.body: ; preds = %inner.body, %inner.ph
- %indvars.iv35 = phi i64 [ %indvars.iv38, %inner.ph ], [ %indvars.iv.next36, %inner.body ]
- %2 = add nsw i64 %indvars.iv35, %1
- %arrayidx = getelementptr inbounds i32, ptr %b, i64 %2
- %3 = load i32, ptr %arrayidx, align 4, !tbaa !2
- %mul8 = mul nsw i32 %3, %3
- %arrayidx12 = getelementptr inbounds i32, ptr %a, i64 %2
- store i32 %mul8, ptr %arrayidx12, align 4, !tbaa !2
- %indvars.iv.next36 = add nuw nsw i64 %indvars.iv35, 1
- %exitcond = icmp eq i64 %indvars.iv.next36, %wide.trip.count
- br i1 %exitcond, label %outer.inc, label %inner.body
-
-outer.inc: ; preds = %inner.body, %outer.body
- %indvars.iv.next39 = add nuw nsw i64 %indvars.iv38, 1
- %exitcond42 = icmp eq i64 %indvars.iv.next39, %wide.trip.count41
- br i1 %exitcond42, label %for.end15, label %outer.body, !llvm.loop !6
-
-for.end15: ; preds = %outer.inc, %entry
- ret void
-}
-
-
-; Case 2 (for (j = 0; j < i; j++)): Inner loop with divergent upper-bound.
-
-; CHECK-LABEL: loop_ub
-; CHECK: LV: Not vectorizing: Outer loop contains divergent loops.
-; CHECK: LV: Not vectorizing: Unsupported outer loop.
-
-define void @loop_ub(ptr nocapture %a, ptr nocapture readonly %b, i32 %N, i32 %M) {
-entry:
- %cmp32 = icmp sgt i32 %N, 0
- br i1 %cmp32, label %outer.ph, label %for.end15
-
-outer.ph: ; preds = %entry
- %0 = sext i32 %M to i64
- %wide.trip.count41 = zext i32 %N to i64
- br label %outer.body
-
-outer.body: ; preds = %outer.inc, %outer.ph
- %indvars.iv38 = phi i64 [ 0, %outer.ph ], [ %indvars.iv.next39, %outer.inc ]
- %cmp230 = icmp eq i64 %indvars.iv38, 0
- br i1 %cmp230, label %outer.inc, label %inner.ph
-
-inner.ph: ; preds = %outer.body
- %1 = mul nsw i64 %indvars.iv38, %0
- br label %inner.body
-
-inner.body: ; preds = %inner.body, %inner.ph
- %indvars.iv = phi i64 [ 0, %inner.ph ], [ %indvars.iv.next, %inner.body ]
- %2 = add nsw i64 %indvars.iv, %1
- %arrayidx = getelementptr inbounds i32, ptr %b, i64 %2
- %3 = load i32, ptr %arrayidx, align 4, !tbaa !2
- %mul8 = mul nsw i32 %3, %3
- %arrayidx12 = getelementptr inbounds i32, ptr %a, i64 %2
- store i32 %mul8, ptr %arrayidx12, align 4, !tbaa !2
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %exitcond = icmp eq i64 %indvars.iv.next, %indvars.iv38
- br i1 %exitcond, label %outer.inc, label %inner.body
-
-outer.inc: ; preds = %inner.body, %outer.body
- %indvars.iv.next39 = add nuw nsw i64 %indvars.iv38, 1
- %exitcond42 = icmp eq i64 %indvars.iv.next39, %wide.trip.count41
- br i1 %exitcond42, label %for.end15, label %outer.body, !llvm.loop !6
-
-for.end15: ; preds = %outer.inc, %entry
- ret void
-}
-
-; Case 3 (for (j = 0; j < M; j+=i)): Inner loop with divergent step.
-
-; CHECK-LABEL: iv_step
-; CHECK: LV: Not vectorizing: Outer loop contains divergent loops.
-; CHECK: LV: Not vectorizing: Unsupported outer loop.
-
-define void @iv_step(ptr nocapture %a, ptr nocapture readonly %b, i32 %N, i32 %M) {
-entry:
- %cmp33 = icmp sgt i32 %N, 0
- br i1 %cmp33, label %outer.ph, label %for.end15
-
-outer.ph: ; preds = %entry
- %cmp231 = icmp sgt i32 %M, 0
- %0 = sext i32 %M to i64
- %wide.trip.count = zext i32 %N to i64
- br label %outer.body
-
-outer.body: ; preds = %for.inc14, %outer.ph
- %indvars.iv39 = phi i64 [ 0, %outer.ph ], [ %indvars.iv.next40, %for.inc14 ]
- br i1 %cmp231, label %inner.ph, label %for.inc14
-
-inner.ph: ; preds = %outer.body
- %1 = mul nsw i64 %indvars.iv39, %0
- br label %inner.body
-
-inner.body: ; preds = %inner.ph, %inner.body
- %indvars.iv36 = phi i64 [ 0, %inner.ph ], [ %indvars.iv.next37, %inner.body ]
- %2 = add nsw i64 %indvars.iv36, %1
- %arrayidx = getelementptr inbounds i32, ptr %b, i64 %2
- %3 = load i32, ptr %arrayidx, align 4, !tbaa !2
- %mul8 = mul nsw i32 %3, %3
- %arrayidx12 = getelementptr inbounds i32, ptr %a, i64 %2
- store i32 %mul8, ptr %arrayidx12, align 4, !tbaa !2
- %indvars.iv.next37 = add nuw nsw i64 %indvars.iv36, %indvars.iv39
- %cmp2 = icmp slt i64 %indvars.iv.next37, %0
- br i1 %cmp2, label %inner.body, label %for.inc14
-
-for.inc14: ; preds = %inner.body, %outer.body
- %indvars.iv.next40 = add nuw nsw i64 %indvars.iv39, 1
- %exitcond = icmp eq i64 %indvars.iv.next40, %wide.trip.count
- br i1 %exitcond, label %for.end15, label %outer.body, !llvm.loop !6
-
-for.end15: ; preds = %for.inc14, %entry
- ret void
-}
-
-!llvm.module.flags = !{!0}
-!llvm.ident = !{!1}
-
-!0 = !{i32 1, !"wchar_size", i32 4}
-!1 = !{!"clang version 6.0.0"}
-!2 = !{!3, !3, i64 0}
-!3 = !{!"int", !4, i64 0}
-!4 = !{!"omnipotent char", !5, i64 0}
-!5 = !{!"Simple C/C++ TBAA"}
-!6 = distinct !{!6, !7, !8}
-!7 = !{!"llvm.loop.vectorize.width", i32 8}
-!8 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll b/llvm/test/Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll
deleted file mode 100644
index 932a6e4ba88d1..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll
+++ /dev/null
@@ -1,136 +0,0 @@
-; RUN: opt < %s -passes=loop-vectorize -enable-vplan-native-path -debug-only=loop-vectorize --disable-output -S 2>&1 | FileCheck %s
-; REQUIRES: asserts
-
-; Verify that LV can handle explicit vectorization outer loops with uniform branches
-; but bails out on outer loops with divergent branches.
-
-; Root C/C++ source code for the test cases
-; void foo(int *a, int *b, int N, int M)
-; {
-; int i, j;
-; #pragma clang loop vectorize(enable) vectorize_width(8)
-; for (i = 0; i < N; i++) {
-; // Tested conditional branch. COND will be replaced per test.
-; if (COND)
-; for (j = 0; j < M; j++) {
-; a[i*M+j] = bptr b[i*M+j];
-; }
-; }
-; }
-
-; Case 1 (COND => M == N): Outer loop with uniform conditional branch.
-
-; CHECK-LABEL: uniform_branch
-; CHECK: LV: We can vectorize this outer loop!
-
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-
-define void @uniform_branch(ptr nocapture %a, ptr nocapture readonly %b, i32 %N, i32 %M) {
-entry:
- %cmp39 = icmp sgt i32 %N, 0
- br i1 %cmp39, label %outer.ph, label %for.end19
-
-outer.ph: ; preds = %entry
- %cmp337 = icmp slt i32 %M, 1
- %0 = sext i32 %M to i64
- %N64 = zext i32 %N to i64
- %M64 = zext i32 %M to i64
- %cmp1 = icmp ne i32 %M, %N ; Uniform condition
- %brmerge = or i1 %cmp1, %cmp337 ; Uniform condition
- br label %outer.body
-
-outer.body: ; preds = %outer.inc, %outer.ph
- %indvars.iv42 = phi i64 [ 0, %outer.ph ], [ %indvars.iv.next43, %outer.inc ]
- %1 = mul nsw i64 %indvars.iv42, %0
- %arrayidx = getelementptr inbounds i32, ptr %b, i64 %1
- %2 = load i32, ptr %arrayidx, align 4, !tbaa !2
- br i1 %brmerge, label %outer.inc, label %inner.ph ; Supported uniform branch
-
-inner.ph: ; preds = %outer.body
- br label %inner.body
-
-inner.body: ; preds = %inner.ph, %inner.body
- %indvars.iv = phi i64 [ %indvars.iv.next, %inner.body ], [ 0, %inner.ph ]
- %3 = add nsw i64 %indvars.iv, %1
- %arrayidx7 = getelementptr inbounds i32, ptr %b, i64 %3
- %4 = load i32, ptr %arrayidx7, align 4, !tbaa !2
- %mul12 = mul nsw i32 %4, %4
- %arrayidx16 = getelementptr inbounds i32, ptr %a, i64 %3
- store i32 %mul12, ptr %arrayidx16, align 4, !tbaa !2
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %exitcond = icmp eq i64 %indvars.iv.next, %M64
- br i1 %exitcond, label %outer.inc, label %inner.body
-
-outer.inc: ; preds = %inner.body, %outer.body
- %indvars.iv.next43 = add nuw nsw i64 %indvars.iv42, 1
- %exitcond46 = icmp eq i64 %indvars.iv.next43, %N64
- br i1 %exitcond46, label %for.end19, label %outer.body, !llvm.loop !6
-
-for.end19: ; preds = %outer.inc, %entry
- ret void
-}
-
-
-; Case 2 (COND => B[i * M] == 0): Outer loop with divergent conditional branch.
-
-; CHECK-LABEL: divergent_branch
-; CHECK: Unsupported conditional branch.
-; CHECK: LV: Not vectorizing: Unsupported outer loop.
-
-define void @divergent_branch(ptr nocapture %a, ptr nocapture readonly %b, i32 %N, i32 %M) {
-entry:
- %cmp39 = icmp sgt i32 %N, 0
- br i1 %cmp39, label %outer.ph, label %for.end19
-
-outer.ph: ; preds = %entry
- %cmp337 = icmp slt i32 %M, 1
- %0 = sext i32 %M to i64
- %N64 = zext i32 %N to i64
- %M64 = zext i32 %M to i64
- br label %outer.body
-
-outer.body: ; preds = %outer.inc, %outer.ph
- %indvars.iv42 = phi i64 [ 0, %outer.ph ], [ %indvars.iv.next43, %outer.inc ]
- %1 = mul nsw i64 %indvars.iv42, %0
- %arrayidx = getelementptr inbounds i32, ptr %b, i64 %1
- %2 = load i32, ptr %arrayidx, align 4, !tbaa !2
- %cmp1 = icmp ne i32 %2, 0 ; Divergent condition
- %brmerge = or i1 %cmp1, %cmp337 ; Divergent condition
- br i1 %brmerge, label %outer.inc, label %inner.ph ; Unsupported divergent branch.
-
-inner.ph: ; preds = %outer.body
- br label %inner.body
-
-inner.body: ; preds = %inner.ph, %inner.body
- %indvars.iv = phi i64 [ %indvars.iv.next, %inner.body ], [ 0, %inner.ph ]
- %3 = add nsw i64 %indvars.iv, %1
- %arrayidx7 = getelementptr inbounds i32, ptr %b, i64 %3
- %4 = load i32, ptr %arrayidx7, align 4, !tbaa !2
- %mul12 = mul nsw i32 %4, %4
- %arrayidx16 = getelementptr inbounds i32, ptr %a, i64 %3
- store i32 %mul12, ptr %arrayidx16, align 4, !tbaa !2
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %exitcond = icmp eq i64 %indvars.iv.next, %M64
- br i1 %exitcond, label %outer.inc, label %inner.body
-
-outer.inc: ; preds = %inner.body, %outer.body
- %indvars.iv.next43 = add nuw nsw i64 %indvars.iv42, 1
- %exitcond46 = icmp eq i64 %indvars.iv.next43, %N64
- br i1 %exitcond46, label %for.end19, label %outer.body, !llvm.loop !6
-
-for.end19: ; preds = %outer.inc, %entry
- ret void
-}
-
-!llvm.module.flags = !{!0}
-!llvm.ident = !{!1}
-
-!0 = !{i32 1, !"wchar_size", i32 4}
-!1 = !{!"clang version 6.0.0"}
-!2 = !{!3, !3, i64 0}
-!3 = !{!"int", !4, i64 0}
-!4 = !{!"omnipotent char", !5, i64 0}
-!5 = !{!"Simple C/C++ TBAA"}
-!6 = distinct !{!6, !7, !8}
-!7 = !{!"llvm.loop.vectorize.width", i32 8}
-!8 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/inner-loop-reduction.ll b/llvm/test/Transforms/LoopVectorize/inner-loop-reduction.ll
new file mode 100644
index 0000000000000..8ed9fc24fc543
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/inner-loop-reduction.ll
@@ -0,0 +1,86 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
+
+; Vectorize explict marked outer loop using vplan native path. Inner loop
+; contains simple double add reduction. IR is compiled and modified by hand
+; from following C code:
+; void inner_loop_reduction(const ptr restrict in_a, const ptr restrict in_b, ptr restrict out)
+; {
+; #pragma clang loop vectorize(enable)
+; for (int i = 0; i < 1000; ++i) {
+; double a = in_a[i];
+; double b = in_b[i];
+; for (int j = 0; j < 10000; ++j) {
+; a = a + b;
+; }
+; out[i] = a;
+; }
+; }
+define void @inner_loop_reduction(ptr noalias %a.in, ptr noalias %b.in, ptr noalias %c.out) {
+; CHECK-LABEL: define void @inner_loop_reduction(
+; CHECK-SAME: ptr noalias [[A_IN:%.*]], ptr noalias [[B_IN:%.*]], ptr noalias [[C_OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR1_HEADER:.*]]
+; CHECK: [[FOR1_HEADER]]:
+; CHECK-NEXT: [[INDVAR1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVAR11:%.*]], %[[FOR1_LATCH:.*]] ]
+; CHECK-NEXT: [[A_PTR:%.*]] = getelementptr inbounds double, ptr [[A_IN]], i64 [[INDVAR1]]
+; CHECK-NEXT: [[A:%.*]] = load double, ptr [[A_PTR]], align 8
+; CHECK-NEXT: [[B_PTR:%.*]] = getelementptr inbounds double, ptr [[B_IN]], i64 [[INDVAR1]]
+; CHECK-NEXT: [[B:%.*]] = load double, ptr [[B_PTR]], align 8
+; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x double> splat (double -0.000000e+00), double [[A]], i32 0
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[B]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x double> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1]] = fadd <4 x double> [[BROADCAST_SPLAT]], [[VEC_PHI]]
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 10000
+; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP3:%.*]] = call double @llvm.vector.reduce.fadd.v4f64(double -0.000000e+00, <4 x double> [[TMP1]])
+; CHECK-NEXT: br label %[[FOR1_LATCH]]
+; CHECK: [[FOR1_LATCH]]:
+; CHECK-NEXT: [[C_PTR:%.*]] = getelementptr inbounds double, ptr [[C_OUT]], i64 [[INDVAR1]]
+; CHECK-NEXT: store double [[TMP3]], ptr [[C_PTR]], align 8
+; CHECK-NEXT: [[INDVAR11]] = add nuw nsw i64 [[INDVAR1]], 1
+; CHECK-NEXT: [[FOR1_COND:%.*]] = icmp eq i64 [[INDVAR11]], 1000
+; CHECK-NEXT: br i1 [[FOR1_COND]], label %[[EXIT:.*]], label %[[FOR1_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %for1.header
+
+for1.header: ; preds = %entry
+ %indvar1 = phi i64 [ 0, %entry ], [ %indvar11, %for1.latch ]
+ %a.ptr = getelementptr inbounds double, ptr %a.in, i64 %indvar1
+ %a = load double, ptr %a.ptr, align 8
+ %b.ptr = getelementptr inbounds double, ptr %b.in, i64 %indvar1
+ %b = load double, ptr %b.ptr, align 8
+ br label %for2.header
+
+for2.header: ; preds = %for1.header, %for2.header
+ %indvar2 = phi i32 [ 0, %for1.header ], [ %indvar21, %for2.header ]
+ %a.reduction = phi double [ %a, %for1.header ], [ %a.reduction1, %for2.header ]
+ %a.reduction1 = fadd double %b, %a.reduction
+ %indvar21 = add nuw nsw i32 %indvar2, 1
+ %for2.cond = icmp eq i32 %indvar21, 10000
+ br i1 %for2.cond, label %for1.latch, label %for2.header
+
+for1.latch: ; preds = %for2.header
+ %c.ptr = getelementptr inbounds double, ptr %c.out, i64 %indvar1
+ store double %a.reduction1, ptr %c.ptr, align 8
+ %indvar11 = add nuw nsw i64 %indvar1, 1
+ %for1.cond = icmp eq i64 %indvar11, 1000
+ br i1 %for1.cond, label %exit, label %for1.header, !llvm.loop !0
+
+exit: ; preds = %for1.latch
+ ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll b/llvm/test/Transforms/LoopVectorize/inner-loop-with-rtchecks.ll
similarity index 98%
rename from llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
rename to llvm/test/Transforms/LoopVectorize/inner-loop-with-rtchecks.ll
index 22cf860c8b58c..110a8b2d742af 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-native-path-inner-loop-with-runtime-checks.ll
+++ b/llvm/test/Transforms/LoopVectorize/inner-loop-with-rtchecks.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -p loop-vectorize -enable-vplan-native-path -force-vector-width=4 -S %s | FileCheck %s
+; RUN: opt -p loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32"
diff --git a/llvm/test/Transforms/LoopVectorize/noalias-scope-decl-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/noalias-scope-decl-outer-loop.ll
deleted file mode 100644
index 80667d6d8d288..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/noalias-scope-decl-outer-loop.ll
+++ /dev/null
@@ -1,73 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt -S -passes='loop-vectorize,verify' -enable-vplan-native-path -force-vector-width=2 < %s | FileCheck %s
-
-; The noalias.scope.decl intrinsic declares a noalias scope valid for a single
-; iteration. During outer loop vectorization, emitting it as a single-scalar
-; replicate would incorrectly extend the scope across multiple original
-; iterations packed into one vector iteration. Bail out of vectorization for
-; outer loops containing noalias.scope.decl.
-; FIXME: We could still vectorize by dropping the noalias.scope.decl and
-; stripping !alias.scope/!noalias metadata from the loop body.
-
-define void @test_noalias_scope_decl(ptr %a, ptr %b, i64 %n) {
-; CHECK-LABEL: define void @test_noalias_scope_decl(
-; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[SCALAR_PH:.*]]:
-; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
-; CHECK: [[OUTER_HEADER]]:
-; CHECK-NEXT: [[J:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
-; CHECK-NEXT: call void @llvm.experimental.noalias.scope.decl(metadata [[META0:![0-9]+]])
-; CHECK-NEXT: br label %[[INNER:.*]]
-; CHECK: [[INNER]]:
-; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
-; CHECK-NEXT: [[IDX:%.*]] = add i64 [[J]], [[K]]
-; CHECK-NEXT: [[PTR_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDX]]
-; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[PTR_A]], align 4, !alias.scope [[META0]]
-; CHECK-NEXT: [[PTR_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IDX]]
-; CHECK-NEXT: store i32 [[VAL]], ptr [[PTR_B]], align 4, !noalias [[META0]]
-; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
-; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH]], label %[[INNER]]
-; CHECK: [[OUTER_LATCH]]:
-; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
-; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %outer.header
-
-outer.header:
- %j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
- call void @llvm.experimental.noalias.scope.decl(metadata !3)
- br label %inner
-
-inner:
- %k = phi i64 [ 0, %outer.header ], [ %k.next, %inner ]
- %idx = add i64 %j, %k
- %ptr.a = getelementptr inbounds i32, ptr %a, i64 %idx
- %val = load i32, ptr %ptr.a, align 4, !alias.scope !3
- %ptr.b = getelementptr inbounds i32, ptr %b, i64 %idx
- store i32 %val, ptr %ptr.b, align 4, !noalias !3
- %k.next = add i64 %k, 1
- %inner.cond = icmp eq i64 %k.next, %n
- br i1 %inner.cond, label %outer.latch, label %inner
-
-outer.latch:
- %j.next = add i64 %j, 1
- %outer.cond = icmp eq i64 %j.next, %n
- br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
-
-exit:
- ret void
-}
-
-declare void @llvm.experimental.noalias.scope.decl(metadata)
-
-!0 = distinct !{!0, !1, !2}
-!1 = !{!"llvm.loop.mustprogress"}
-!2 = !{!"llvm.loop.vectorize.enable", i1 true}
-!3 = !{!4}
-!4 = distinct !{!4, !5}
-!5 = distinct !{!5}
diff --git a/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll
deleted file mode 100644
index 76da1de53311f..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/non-widenable-intrinsics-outer-loop.ll
+++ /dev/null
@@ -1,477 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -S -passes='loop-vectorize,verify' -enable-vplan-native-path -force-vector-width=2 < %s | FileCheck %s
-
-; Test that non-widenable intrinsics in the outer loop are emitted as replicate
-; recipe and do not crash the VPlan native path.
-
-; Check that llvm.assume is replicated per lane for the assumption.
-define void @test_assume(ptr %arr, i64 %n) {
-; CHECK-LABEL: define void @test_assume(
-; CHECK-SAME: ptr [[ARR:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[N]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = icmp sgt <2 x i64> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP0]], i32 0
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP0]], i32 1
-; CHECK-NEXT: call void @llvm.assume(i1 [[TMP9]])
-; CHECK-NEXT: call void @llvm.assume(i1 [[TMP8]])
-; CHECK-NEXT: br label %[[INNER1:.*]]
-; CHECK: [[INNER1]]:
-; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER1]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = add <2 x i64> [[VEC_IND]], [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[ARR]], <2 x i64> [[TMP2]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> splat (i32 1), <2 x ptr> align 4 [[TMP3]], <2 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP4]] = add <2 x i64> [[TMP1]], splat (i64 1)
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <2 x i64> [[TMP4]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0
-; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[INNER1]]
-; CHECK: [[VECTOR_LATCH]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
-; CHECK: [[OUTER_HEADER]]:
-; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
-; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i64 [[J]], 0
-; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
-; CHECK-NEXT: br label %[[INNER:.*]]
-; CHECK: [[INNER]]:
-; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
-; CHECK-NEXT: [[IDX:%.*]] = add i64 [[J]], [[K]]
-; CHECK-NEXT: [[PTR:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i64 [[IDX]]
-; CHECK-NEXT: store i32 1, ptr [[PTR]], align 4
-; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
-; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH]], label %[[INNER]]
-; CHECK: [[OUTER_LATCH]]:
-; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
-; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %outer.header
-
-outer.header:
- %j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
- %cmp = icmp sgt i64 %j, 0
- call void @llvm.assume(i1 %cmp)
- br label %inner
-
-inner:
- %k = phi i64 [ 0, %outer.header ], [ %k.next, %inner ]
- %idx = add i64 %j, %k
- %ptr = getelementptr inbounds i32, ptr %arr, i64 %idx
- store i32 1, ptr %ptr, align 4
- %k.next = add i64 %k, 1
- %inner.cond = icmp eq i64 %k.next, %n
- br i1 %inner.cond, label %outer.latch, label %inner
-
-outer.latch:
- %j.next = add i64 %j, 1
- %outer.cond = icmp eq i64 %j.next, %n
- br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
-
-exit:
- ret void
-}
-
-; Check that llvm.assume with a noncanonical induction variable is replicated per lane.
-define void @test_assume_non_canonical_iv(ptr %arr, i64 %n) {
-; CHECK-LABEL: define void @test_assume_non_canonical_iv(
-; CHECK-SAME: ptr [[ARR:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[N_VEC]], 3
-; CHECK-NEXT: [[TMP1:%.*]] = add i64 10, [[TMP0]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[N]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 10, i64 13>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = icmp sge <2 x i64> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP2]], i32 1
-; CHECK-NEXT: call void @llvm.assume(i1 [[TMP3]])
-; CHECK-NEXT: call void @llvm.assume(i1 [[TMP4]])
-; CHECK-NEXT: br label %[[INNER1:.*]]
-; CHECK: [[INNER1]]:
-; CHECK-NEXT: [[TMP5:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP10:%.*]], %[[INNER1]] ]
-; CHECK-NEXT: [[TMP6:%.*]] = add <2 x i64> [[VEC_IND]], [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[ARR]], <2 x i64> [[TMP6]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> align 4 [[TMP7]], <2 x i1> splat (i1 true), <2 x i32> poison)
-; CHECK-NEXT: [[TMP8:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> splat (i32 1), <2 x i32> splat (i32 -1)
-; CHECK-NEXT: [[TMP9:%.*]] = add <2 x i32> [[WIDE_MASKED_GATHER]], [[TMP8]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> [[TMP9]], <2 x ptr> align 4 [[TMP7]], <2 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP10]] = add <2 x i64> [[TMP5]], splat (i64 1)
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq <2 x i64> [[TMP10]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i1> [[TMP11]], i32 0
-; CHECK-NEXT: br i1 [[TMP12]], label %[[VECTOR_LATCH]], label %[[INNER1]]
-; CHECK: [[VECTOR_LATCH]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 6)
-; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi i64 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ 10, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
-; CHECK: [[OUTER_HEADER]]:
-; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
-; CHECK-NEXT: [[M:%.*]] = phi i64 [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[M_NEXT:%.*]], %[[OUTER_LATCH]] ]
-; CHECK-NEXT: [[M_NEXT]] = add i64 [[M]], 3
-; CHECK-NEXT: [[CMP:%.*]] = icmp sge i64 [[M]], 0
-; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
-; CHECK-NEXT: br label %[[INNER:.*]]
-; CHECK: [[INNER]]:
-; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
-; CHECK-NEXT: [[IDX:%.*]] = add i64 [[M]], [[K]]
-; CHECK-NEXT: [[PTR:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i64 [[IDX]]
-; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[PTR]], align 4
-; CHECK-NEXT: [[INC:%.*]] = select i1 [[CMP]], i32 1, i32 -1
-; CHECK-NEXT: [[VAL2:%.*]] = add i32 [[VAL]], [[INC]]
-; CHECK-NEXT: store i32 [[VAL2]], ptr [[PTR]], align 4
-; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
-; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH]], label %[[INNER]]
-; CHECK: [[OUTER_LATCH]]:
-; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
-; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %outer.header
-
-outer.header:
- %j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
- %m = phi i64 [ 10, %entry ], [ %m.next, %outer.latch ]
- %m.next = add i64 %m, 3
- %cmp = icmp sge i64 %m, 0
- call void @llvm.assume(i1 %cmp)
- br label %inner
-
-inner:
- %k = phi i64 [ 0, %outer.header ], [ %k.next, %inner ]
- %idx = add i64 %m, %k
- %ptr = getelementptr inbounds i32, ptr %arr, i64 %idx
- %val = load i32, ptr %ptr, align 4
- %inc = select i1 %cmp, i32 1, i32 -1
- %val2 = add i32 %val, %inc
- store i32 %val2, ptr %ptr, align 4
- %k.next = add i64 %k, 1
- %inner.cond = icmp eq i64 %k.next, %n
- br i1 %inner.cond, label %outer.latch, label %inner
-
-outer.latch:
- %j.next = add i64 %j, 1
- %outer.cond = icmp eq i64 %j.next, %n
- br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
-
-exit:
- ret void
-}
-
-define void @test_lifetime(ptr %arr, i64 %n) {
-; CHECK-LABEL: define void @test_lifetime(
-; CHECK-SAME: ptr [[ARR:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[N]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[A]])
-; CHECK-NEXT: br label %[[INNER1:.*]]
-; CHECK: [[INNER1]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER1]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i64> [[VEC_IND]], [[TMP0]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[ARR]], <2 x i64> [[TMP1]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> splat (i32 1), <2 x ptr> align 4 [[TMP2]], <2 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP3]] = add <2 x i64> [[TMP0]], splat (i64 1)
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <2 x i64> [[TMP3]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
-; CHECK-NEXT: br i1 [[TMP5]], label %[[VECTOR_LATCH]], label %[[INNER1]]
-; CHECK: [[VECTOR_LATCH]]:
-; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[A]])
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
-; CHECK: [[OUTER_HEADER]]:
-; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
-; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[A]])
-; CHECK-NEXT: br label %[[INNER:.*]]
-; CHECK: [[INNER]]:
-; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
-; CHECK-NEXT: [[IDX:%.*]] = add i64 [[J]], [[K]]
-; CHECK-NEXT: [[PTR:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i64 [[IDX]]
-; CHECK-NEXT: store i32 1, ptr [[PTR]], align 4
-; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
-; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH]], label %[[INNER]]
-; CHECK: [[OUTER_LATCH]]:
-; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[A]])
-; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
-; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- %a = alloca i32
- br label %outer.header
-
-outer.header:
- %j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
- call void @llvm.lifetime.start.p0(i64 4, ptr %a)
- br label %inner
-
-inner:
- %k = phi i64 [ 0, %outer.header ], [ %k.next, %inner ]
- %idx = add i64 %j, %k
- %ptr = getelementptr inbounds i32, ptr %arr, i64 %idx
- store i32 1, ptr %ptr, align 4
- %k.next = add i64 %k, 1
- %inner.cond = icmp eq i64 %k.next, %n
- br i1 %inner.cond, label %outer.latch, label %inner
-
-outer.latch:
- call void @llvm.lifetime.end.p0(i64 4, ptr %a)
- %j.next = add i64 %j, 1
- %outer.cond = icmp eq i64 %j.next, %n
- br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
-
-exit:
- ret void
-}
-
-define void @test_sideeffect(ptr %arr, i64 %n) {
-; CHECK-LABEL: define void @test_sideeffect(
-; CHECK-SAME: ptr [[ARR:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[N]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; CHECK-NEXT: call void @llvm.sideeffect()
-; CHECK-NEXT: br label %[[INNER1:.*]]
-; CHECK: [[INNER1]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER1]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i64> [[VEC_IND]], [[TMP0]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[ARR]], <2 x i64> [[TMP1]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> splat (i32 1), <2 x ptr> align 4 [[TMP2]], <2 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP3]] = add <2 x i64> [[TMP0]], splat (i64 1)
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <2 x i64> [[TMP3]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
-; CHECK-NEXT: br i1 [[TMP5]], label %[[VECTOR_LATCH]], label %[[INNER1]]
-; CHECK: [[VECTOR_LATCH]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
-; CHECK: [[OUTER_HEADER]]:
-; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
-; CHECK-NEXT: call void @llvm.sideeffect()
-; CHECK-NEXT: br label %[[INNER:.*]]
-; CHECK: [[INNER]]:
-; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
-; CHECK-NEXT: [[IDX:%.*]] = add i64 [[J]], [[K]]
-; CHECK-NEXT: [[PTR:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i64 [[IDX]]
-; CHECK-NEXT: store i32 1, ptr [[PTR]], align 4
-; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
-; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH]], label %[[INNER]]
-; CHECK: [[OUTER_LATCH]]:
-; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
-; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP10:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %outer.header
-
-outer.header:
- %j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
- call void @llvm.sideeffect()
- br label %inner
-
-inner:
- %k = phi i64 [ 0, %outer.header ], [ %k.next, %inner ]
- %idx = add i64 %j, %k
- %ptr = getelementptr inbounds i32, ptr %arr, i64 %idx
- store i32 1, ptr %ptr, align 4
- %k.next = add i64 %k, 1
- %inner.cond = icmp eq i64 %k.next, %n
- br i1 %inner.cond, label %outer.latch, label %inner
-
-outer.latch:
- %j.next = add i64 %j, 1
- %outer.cond = icmp eq i64 %j.next, %n
- br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
-
-exit:
- ret void
-}
-
-; Check that llvm.pseudoprobe is replicated once per lane for accurate sample trip count.
-define void @test_pseudoprobe(ptr %arr, i64 %n) {
-; CHECK-LABEL: define void @test_pseudoprobe(
-; CHECK-SAME: ptr [[ARR:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 2
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[N]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; CHECK-NEXT: call void @llvm.pseudoprobe(i64 6699318081062747564, i64 1, i32 0, i64 -1)
-; CHECK-NEXT: call void @llvm.pseudoprobe(i64 6699318081062747564, i64 1, i32 0, i64 -1)
-; CHECK-NEXT: br label %[[INNER1:.*]]
-; CHECK: [[INNER1]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER1]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i64> [[VEC_IND]], [[TMP0]]
-; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[ARR]], <2 x i64> [[TMP1]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> splat (i32 1), <2 x ptr> align 4 [[TMP2]], <2 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP3]] = add <2 x i64> [[TMP0]], splat (i64 1)
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <2 x i64> [[TMP3]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
-; CHECK-NEXT: br i1 [[TMP5]], label %[[VECTOR_LATCH]], label %[[INNER1]]
-; CHECK: [[VECTOR_LATCH]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
-; CHECK: [[OUTER_HEADER]]:
-; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[J_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
-; CHECK-NEXT: call void @llvm.pseudoprobe(i64 6699318081062747564, i64 1, i32 0, i64 -1)
-; CHECK-NEXT: br label %[[INNER:.*]]
-; CHECK: [[INNER]]:
-; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[K_NEXT:%.*]], %[[INNER]] ]
-; CHECK-NEXT: [[IDX:%.*]] = add i64 [[J]], [[K]]
-; CHECK-NEXT: [[PTR:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i64 [[IDX]]
-; CHECK-NEXT: store i32 1, ptr [[PTR]], align 4
-; CHECK-NEXT: [[K_NEXT]] = add i64 [[K]], 1
-; CHECK-NEXT: [[INNER_COND:%.*]] = icmp eq i64 [[K_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[INNER_COND]], label %[[OUTER_LATCH]], label %[[INNER]]
-; CHECK: [[OUTER_LATCH]]:
-; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1
-; CHECK-NEXT: [[OUTER_COND:%.*]] = icmp eq i64 [[J_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[OUTER_COND]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP12:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %outer.header
-
-outer.header:
- %j = phi i64 [ 0, %entry ], [ %j.next, %outer.latch ]
- call void @llvm.pseudoprobe(i64 6699318081062747564, i64 1, i32 0, i64 -1)
- br label %inner
-
-inner:
- %k = phi i64 [ 0, %outer.header ], [ %k.next, %inner ]
- %idx = add i64 %j, %k
- %ptr = getelementptr inbounds i32, ptr %arr, i64 %idx
- store i32 1, ptr %ptr, align 4
- %k.next = add i64 %k, 1
- %inner.cond = icmp eq i64 %k.next, %n
- br i1 %inner.cond, label %outer.latch, label %inner
-
-outer.latch:
- %j.next = add i64 %j, 1
- %outer.cond = icmp eq i64 %j.next, %n
- br i1 %outer.cond, label %exit, label %outer.header, !llvm.loop !0
-
-exit:
- ret void
-}
-
-declare void @llvm.assume(i1)
-declare void @llvm.lifetime.start.p0(i64, ptr)
-declare void @llvm.lifetime.end.p0(i64, ptr)
-declare void @llvm.sideeffect()
-declare void @llvm.pseudoprobe(i64, i64, i32, i64)
-
-!0 = distinct !{!0, !1, !2}
-!1 = !{!"llvm.loop.mustprogress"}
-!2 = !{!"llvm.loop.vectorize.enable", i1 true}
-;.
-; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
-; CHECK: [[META1]] = !{!"llvm.loop.mustprogress"}
-; CHECK: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META3]], [[META2]]}
-; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]], [[META3]]}
-; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META3]], [[META2]]}
-; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]], [[META3]]}
-; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META3]], [[META2]]}
-; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]], [[META3]]}
-; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META3]], [[META2]]}
-; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]], [[META2]], [[META3]]}
-; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META3]], [[META2]]}
-;.
diff --git a/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll b/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll
deleted file mode 100644
index 19e10d3c84024..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll
+++ /dev/null
@@ -1,210 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
-; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -S %s | FileCheck %s
-
- at A = common global [1024 x i64] zeroinitializer, align 16
- at B = common global [1024 x i64] zeroinitializer, align 16
-
-define void @inner_latch_header_first_successor(i64 %N, i32 %c, i64 %M) {
-; CHECK-LABEL: define void @inner_latch_header_first_successor(
-; CHECK-SAME: i64 [[N:%.*]], i32 [[C:%.*]], i64 [[M:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[M]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[M]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[M]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [1024 x i64], ptr @A, i64 0, <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[BROADCAST_SPLAT]], <4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: br label %[[INNER3:.*]]
-; CHECK: [[INNER3]]:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER3]] ]
-; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i64> [ [[WIDE_MASKED_GATHER]], %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER3]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [1024 x i64], ptr @B, i64 0, <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 4 [[TMP1]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i64> [[WIDE_MASKED_GATHER5]], [[VEC_PHI4]]
-; CHECK-NEXT: [[TMP3]] = add nsw <4 x i64> [[TMP2]], [[VEC_PHI4]]
-; CHECK-NEXT: [[TMP4]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP4]], [[BROADCAST_SPLAT2]]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0
-; CHECK-NEXT: br i1 [[TMP9]], label %[[VECTOR_LATCH]], label %[[INNER3]]
-; CHECK: [[VECTOR_LATCH]]:
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[TMP3]], <4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[M]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
-; CHECK: [[OUTER_HEADER]]:
-; CHECK-NEXT: [[IV_OUTER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_OUTER_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
-; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 [[IV_OUTER]]
-; CHECK-NEXT: [[RED_START:%.*]] = load i64, ptr [[GEP_A]], align 4
-; CHECK-NEXT: store i32 [[C]], ptr [[GEP_A]], align 4
-; CHECK-NEXT: br label %[[INNER:.*]]
-; CHECK: [[INNER]]:
-; CHECK-NEXT: [[IV_INNER:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[IV_INNER_NEXT:%.*]], %[[INNER]] ]
-; CHECK-NEXT: [[RED:%.*]] = phi i64 [ [[RED_START]], %[[OUTER_HEADER]] ], [ [[RED_NEXT:%.*]], %[[INNER]] ]
-; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds [1024 x i64], ptr @B, i64 0, i64 [[IV_INNER]]
-; CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr [[GEP_B]], align 4
-; CHECK-NEXT: [[ADD:%.*]] = add nsw i64 [[TMP8]], [[RED]]
-; CHECK-NEXT: [[RED_NEXT]] = add nsw i64 [[ADD]], [[RED]]
-; CHECK-NEXT: [[IV_INNER_NEXT]] = add nuw nsw i64 [[IV_INNER]], 1
-; CHECK-NEXT: [[INNER_EC:%.*]] = icmp ne i64 [[IV_INNER_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[INNER_EC]], label %[[INNER]], label %[[OUTER_LATCH]]
-; CHECK: [[OUTER_LATCH]]:
-; CHECK-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i64 [ [[RED_NEXT]], %[[INNER]] ]
-; CHECK-NEXT: store i64 [[RED_NEXT_LCSSA]], ptr [[GEP_A]], align 4
-; CHECK-NEXT: [[IV_OUTER_NEXT]] = add nuw nsw i64 [[IV_OUTER]], 1
-; CHECK-NEXT: [[OUTER_EC:%.*]] = icmp eq i64 [[IV_OUTER_NEXT]], [[M]]
-; CHECK-NEXT: br i1 [[OUTER_EC]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %outer.header
-
-outer.header:
- %iv.outer = phi i64 [ 0, %entry ], [ %iv.outer.next, %outer.latch ]
- %gep.A = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.outer
- %red.start = load i64 , ptr %gep.A, align 4
- store i32 %c, ptr %gep.A, align 4
- br label %inner
-
-inner:
- %iv.inner = phi i64 [ 0, %outer.header ], [ %iv.inner.next, %inner ]
- %red = phi i64 [ %red.start, %outer.header ], [ %red.next, %inner ]
- %gep.B = getelementptr inbounds [1024 x i64], ptr @B, i64 0, i64 %iv.inner
- %2 = load i64, ptr %gep.B, align 4
- %add = add nsw i64 %2, %red
- %red.next = add nsw i64 %add, %red
- %iv.inner.next = add nuw nsw i64 %iv.inner, 1
- %inner.ec = icmp ne i64 %iv.inner.next, %N
- br i1 %inner.ec, label %inner, label %outer.latch
-
-outer.latch:
- store i64 %red.next, ptr %gep.A, align 4
- %iv.outer.next = add nuw nsw i64 %iv.outer, 1
- %outer.ec = icmp eq i64 %iv.outer.next, %M
- br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !1
-
-exit:
- ret void
-}
-
-
-define void @inner_latch_header_second_successor(i64 %N, i32 %c, i64 %M) {
-; CHECK-LABEL: define void @inner_latch_header_second_successor(
-; CHECK-SAME: i64 [[N:%.*]], i32 [[C:%.*]], i64 [[M:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[M]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[M]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[M]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [1024 x i64], ptr @A, i64 0, <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[BROADCAST_SPLAT]], <4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: br label %[[INNER3:.*]]
-; CHECK: [[INNER3]]:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER3]] ]
-; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <4 x i64> [ [[WIDE_MASKED_GATHER]], %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER3]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [1024 x i64], ptr @B, i64 0, <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 4 [[TMP1]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i64> [[WIDE_MASKED_GATHER5]], [[VEC_PHI4]]
-; CHECK-NEXT: [[TMP3]] = add nsw <4 x i64> [[TMP2]], [[VEC_PHI4]]
-; CHECK-NEXT: [[TMP4]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], [[BROADCAST_SPLAT2]]
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0
-; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[INNER3]]
-; CHECK: [[VECTOR_LATCH]]:
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[TMP3]], <4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[M]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
-; CHECK: [[OUTER_HEADER]]:
-; CHECK-NEXT: [[IV_OUTER:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_OUTER_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
-; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 [[IV_OUTER]]
-; CHECK-NEXT: [[RED_START:%.*]] = load i64, ptr [[GEP_A]], align 4
-; CHECK-NEXT: store i32 [[C]], ptr [[GEP_A]], align 4
-; CHECK-NEXT: br label %[[INNER:.*]]
-; CHECK: [[INNER]]:
-; CHECK-NEXT: [[IV_INNER:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[IV_INNER_NEXT:%.*]], %[[INNER]] ]
-; CHECK-NEXT: [[RED:%.*]] = phi i64 [ [[RED_START]], %[[OUTER_HEADER]] ], [ [[RED_NEXT:%.*]], %[[INNER]] ]
-; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds [1024 x i64], ptr @B, i64 0, i64 [[IV_INNER]]
-; CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr [[GEP_B]], align 4
-; CHECK-NEXT: [[ADD:%.*]] = add nsw i64 [[TMP8]], [[RED]]
-; CHECK-NEXT: [[RED_NEXT]] = add nsw i64 [[ADD]], [[RED]]
-; CHECK-NEXT: [[IV_INNER_NEXT]] = add nuw nsw i64 [[IV_INNER]], 1
-; CHECK-NEXT: [[INNER_EC:%.*]] = icmp eq i64 [[IV_INNER_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[INNER_EC]], label %[[OUTER_LATCH]], label %[[INNER]]
-; CHECK: [[OUTER_LATCH]]:
-; CHECK-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i64 [ [[RED_NEXT]], %[[INNER]] ]
-; CHECK-NEXT: store i64 [[RED_NEXT_LCSSA]], ptr [[GEP_A]], align 4
-; CHECK-NEXT: [[IV_OUTER_NEXT]] = add nuw nsw i64 [[IV_OUTER]], 1
-; CHECK-NEXT: [[OUTER_EC:%.*]] = icmp eq i64 [[IV_OUTER_NEXT]], [[M]]
-; CHECK-NEXT: br i1 [[OUTER_EC]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %outer.header
-
-outer.header:
- %iv.outer = phi i64 [ 0, %entry ], [ %iv.outer.next, %outer.latch ]
- %gep.A = getelementptr inbounds [1024 x i64], ptr @A, i64 0, i64 %iv.outer
- %red.start = load i64 , ptr %gep.A, align 4
- store i32 %c, ptr %gep.A, align 4
- br label %inner
-
-inner:
- %iv.inner = phi i64 [ 0, %outer.header ], [ %iv.inner.next, %inner ]
- %red = phi i64 [ %red.start, %outer.header ], [ %red.next, %inner ]
- %gep.B = getelementptr inbounds [1024 x i64], ptr @B, i64 0, i64 %iv.inner
- %2 = load i64, ptr %gep.B, align 4
- %add = add nsw i64 %2, %red
- %red.next = add nsw i64 %add, %red
- %iv.inner.next = add nuw nsw i64 %iv.inner, 1
- %inner.ec = icmp eq i64 %iv.inner.next, %N
- br i1 %inner.ec, label %outer.latch, label %inner
-
-outer.latch:
- store i64 %red.next, ptr %gep.A, align 4
- %iv.outer.next = add nuw nsw i64 %iv.outer, 1
- %outer.ec = icmp eq i64 %iv.outer.next, %M
- br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !1
-
-exit:
- ret void
-}
-
-!1 = distinct !{!1, !2, !3}
-!2 = !{!"llvm.loop.vectorize.width", i32 4}
-!3 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll b/llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll
deleted file mode 100644
index 32dbc273cb6b6..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll
+++ /dev/null
@@ -1,121 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -enable-vplan-native-path -passes=loop-vectorize -S %s | FileCheck %s
-
-; Make sure phi nodes are generated correctly, even if the use list order of
-; the predecessors in the scalar code does not match the order in the generated
-; vector blocks.
-
-; Test from PR45958.
-
-define void @test(ptr %src, i64 %n) {
-; CHECK-LABEL: @test(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[LOOP_1_LATCH5:%.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[LOOP_1_LATCH5]] ]
-; CHECK-NEXT: br label [[LOOP_2_HEADER1:%.*]]
-; CHECK: loop.2.header1:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP5:%.*]], [[LOOP_2_LATCH4:%.*]] ]
-; CHECK-NEXT: br label [[LOOP_32:%.*]]
-; CHECK: loop.32:
-; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i64> [ zeroinitializer, [[LOOP_2_HEADER1]] ], [ [[TMP2:%.*]], [[LOOP_32]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [2000 x i32], ptr [[SRC:%.*]], <4 x i64> [[VEC_IND]], <4 x i64> [[VEC_PHI3]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true), <4 x i32> poison)
-; CHECK-NEXT: [[TMP1:%.*]] = mul nsw <4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 10)
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[TMP1]], <4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP2]] = add nuw nsw <4 x i64> [[VEC_PHI3]], splat (i64 1)
-; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i64> [[TMP2]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP3]], i32 0
-; CHECK-NEXT: br i1 [[TMP4]], label [[LOOP_2_LATCH4]], label [[LOOP_32]]
-; CHECK: loop.2.latch3:
-; CHECK-NEXT: [[TMP5]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP5]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0
-; CHECK-NEXT: br i1 [[TMP7]], label [[LOOP_1_LATCH5]], label [[LOOP_2_HEADER1]]
-; CHECK: vector.latch:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT: br label [[LOOP_1_HEADER:%.*]]
-; CHECK: loop.1.header:
-; CHECK-NEXT: [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP_1_LATCH:%.*]] ]
-; CHECK-NEXT: br label [[LOOP_2_HEADER:%.*]]
-; CHECK: loop.2.header:
-; CHECK-NEXT: [[IV_2:%.*]] = phi i64 [ 0, [[LOOP_1_HEADER]] ], [ [[IV_2_NEXT:%.*]], [[LOOP_2_LATCH:%.*]] ]
-; CHECK-NEXT: br label [[LOOP_3:%.*]]
-; CHECK: loop.3:
-; CHECK-NEXT: [[IV_3:%.*]] = phi i64 [ 0, [[LOOP_2_HEADER]] ], [ [[IV_3_NEXT:%.*]], [[LOOP_3]] ]
-; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds [2000 x i32], ptr [[SRC]], i64 [[IV_1]], i64 [[IV_3]]
-; CHECK-NEXT: [[L1:%.*]] = load i32, ptr [[GEP_SRC]], align 4
-; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[L1]], 10
-; CHECK-NEXT: store i32 [[MUL]], ptr [[GEP_SRC]], align 4
-; CHECK-NEXT: [[IV_3_NEXT]] = add nuw nsw i64 [[IV_3]], 1
-; CHECK-NEXT: [[EC_3:%.*]] = icmp eq i64 [[IV_3_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC_3]], label [[LOOP_2_LATCH]], label [[LOOP_3]]
-; CHECK: loop.2.latch:
-; CHECK-NEXT: [[IV_2_NEXT]] = add nuw nsw i64 [[IV_2]], 1
-; CHECK-NEXT: [[EC_2:%.*]] = icmp eq i64 [[IV_2_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC_2]], label [[LOOP_1_LATCH]], label [[LOOP_2_HEADER]]
-; CHECK: loop.1.latch:
-; CHECK-NEXT: [[IV_1_NEXT]] = add nuw nsw i64 [[IV_1]], 1
-; CHECK-NEXT: [[EC_1:%.*]] = icmp eq i64 [[IV_1_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EC_1]], label [[EXIT]], label [[LOOP_1_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: exit:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %loop.1.header
-
-loop.1.header:
- %iv.1 = phi i64 [ 0, %entry ], [ %iv.1.next, %loop.1.latch ]
- br label %loop.2.header
-
-loop.2.header:
- %iv.2 = phi i64 [ 0, %loop.1.header ], [ %iv.2.next, %loop.2.latch ]
- br label %loop.3
-
-loop.3:
- %iv.3 = phi i64 [ 0, %loop.2.header ], [ %iv.3.next, %loop.3 ]
- %gep.src = getelementptr inbounds [2000 x i32], ptr %src, i64 %iv.1, i64 %iv.3
- %l1 = load i32, ptr %gep.src, align 4
- %mul = mul nsw i32 %l1, 10
- store i32 %mul, ptr %gep.src, align 4
- %iv.3.next = add nuw nsw i64 %iv.3, 1
- %ec.3 = icmp eq i64 %iv.3.next, %n
- br i1 %ec.3, label %loop.2.latch, label %loop.3
-
-loop.2.latch:
- %iv.2.next = add nuw nsw i64 %iv.2, 1
- %ec.2 = icmp eq i64 %iv.2.next, %n
- br i1 %ec.2, label %loop.1.latch, label %loop.2.header
-
-loop.1.latch:
- %iv.1.next = add nuw nsw i64 %iv.1, 1
- %ec.1 = icmp eq i64 %iv.1.next, %n
- br i1 %ec.1, label %exit, label %loop.1.header, !llvm.loop !0
-
-exit: ; preds = %loop.1.latch
- ret void
-
-; uselistorder directives
- uselistorder label %loop.3, { 1, 0 }
- uselistorder label %loop.2.header, { 1, 0 }
-}
-
-!0 = distinct !{!0, !1, !2}
-!1 = !{!"llvm.loop.vectorize.width", i32 4}
-!2 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/outer-loop-wide-phis.ll b/llvm/test/Transforms/LoopVectorize/outer-loop-wide-phis.ll
deleted file mode 100644
index 28ffd74b4f7c2..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/outer-loop-wide-phis.ll
+++ /dev/null
@@ -1,220 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
-; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -S %s | FileCheck %s
-
-define void @wide_phi_2_predecessors(ptr noalias %A, ptr noalias %B, i32 %c, i1 %cond) {
-; CHECK-LABEL: define void @wide_phi_2_predecessors(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i32 [[C:%.*]], i1 [[COND:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[BROADCAST_SPLAT]], <4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: br label %[[INNER_HEADER1:.*]]
-; CHECK: [[INNER_HEADER1]]:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER_LATCH3:.*]] ]
-; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER_LATCH3]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[B]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: br i1 [[COND]], label %[[THEN2:.*]], label %[[INNER_LATCH3]]
-; CHECK: [[THEN2]]:
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[TMP1]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; CHECK-NEXT: br label %[[INNER_LATCH3]]
-; CHECK: [[INNER_LATCH3]]:
-; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i64> [ [[WIDE_MASKED_GATHER]], %[[THEN2]] ], [ zeroinitializer, %[[INNER_HEADER1]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i64> [[VEC_PHI5]], [[VEC_IND]]
-; CHECK-NEXT: [[TMP3]] = add nsw <4 x i64> [[TMP2]], [[VEC_PHI2]]
-; CHECK-NEXT: [[TMP4]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], splat (i64 1000)
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0
-; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[INNER_HEADER1]]
-; CHECK: [[VECTOR_LATCH]]:
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[TMP3]], <4 x ptr> align 8 [[TMP0]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
-; CHECK: [[OUTER_HEADER]]:
-; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 1000, %[[SCALAR_PH]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
-; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OUTER_IV]]
-; CHECK-NEXT: store i32 [[C]], ptr [[GEP_A]], align 4
-; CHECK-NEXT: br label %[[INNER_HEADER:.*]]
-; CHECK: [[INNER_HEADER]]:
-; CHECK-NEXT: [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_LATCH:.*]] ]
-; CHECK-NEXT: [[RED:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[RED_NEXT:%.*]], %[[INNER_LATCH]] ]
-; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INNER_IV]]
-; CHECK-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[INNER_LATCH]]
-; CHECK: [[THEN]]:
-; CHECK-NEXT: [[L_B:%.*]] = load i64, ptr [[GEP_B]], align 8
-; CHECK-NEXT: br label %[[INNER_LATCH]]
-; CHECK: [[INNER_LATCH]]:
-; CHECK-NEXT: [[P:%.*]] = phi i64 [ [[L_B]], %[[THEN]] ], [ 0, %[[INNER_HEADER]] ]
-; CHECK-NEXT: [[ADD_1:%.*]] = add nsw i64 [[P]], [[OUTER_IV]]
-; CHECK-NEXT: [[RED_NEXT]] = add nsw i64 [[ADD_1]], [[RED]]
-; CHECK-NEXT: [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
-; CHECK-NEXT: [[INNER_EC:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 1000
-; CHECK-NEXT: br i1 [[INNER_EC]], label %[[OUTER_LATCH]], label %[[INNER_HEADER]]
-; CHECK: [[OUTER_LATCH]]:
-; CHECK-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i64 [ [[RED_NEXT]], %[[INNER_LATCH]] ]
-; CHECK-NEXT: store i64 [[RED_NEXT_LCSSA]], ptr [[GEP_A]], align 8
-; CHECK-NEXT: [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
-; CHECK-NEXT: [[OUTER_EC:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 1000
-; CHECK-NEXT: br i1 [[OUTER_EC]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %outer.header
-
-outer.header: ; preds = %outer.latch, %outer.header.lr.ph
- %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
- %gep.A = getelementptr inbounds i64, ptr %A, i64 %outer.iv
- store i32 %c, ptr %gep.A, align 4
- br label %inner.header
-
-inner.header:
- %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.latch ]
- %red = phi i64 [ 0, %outer.header ], [ %red.next, %inner.latch ]
- %gep.B = getelementptr inbounds i64, ptr %B, i64 %inner.iv
- br i1 %cond, label %then, label %inner.latch
-
-then:
- %l.b = load i64, ptr %gep.B, align 8
- br label %inner.latch
-
-inner.latch:
- %p = phi i64 [ %l.b, %then ], [ 0, %inner.header ]
- %add.1 = add nsw i64 %p, %outer.iv
- %red.next = add nsw i64 %add.1, %red
- %inner.iv.next = add nuw nsw i64 %inner.iv, 1
- %inner.ec = icmp eq i64 %inner.iv.next, 1000
- br i1 %inner.ec, label %outer.latch, label %inner.header
-
-outer.latch:
- store i64 %red.next, ptr %gep.A, align 8
- %outer.iv.next = add nuw nsw i64 %outer.iv, 1
- %outer.ec = icmp eq i64 %outer.iv.next, 1000
- br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !1
-
-exit:
- ret void
-}
-
-define void @wide_phi_2_predecessors_phi_ops_swapped(ptr noalias %A, ptr noalias %B, i32 %c, i1 %cond) {
-; CHECK-LABEL: define void @wide_phi_2_predecessors_phi_ops_swapped(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i32 [[C:%.*]], i1 [[COND:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i64, ptr [[A]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[BROADCAST_SPLAT]], <4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: br label %[[INNER_HEADER1:.*]]
-; CHECK: [[INNER_HEADER1]]:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[INNER_LATCH3:.*]] ]
-; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[INNER_LATCH3]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[B]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: br i1 [[COND]], label %[[THEN2:.*]], label %[[INNER_LATCH3]]
-; CHECK: [[THEN2]]:
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 8 [[TMP1]], <4 x i1> splat (i1 true), <4 x i64> poison)
-; CHECK-NEXT: br label %[[INNER_LATCH3]]
-; CHECK: [[INNER_LATCH3]]:
-; CHECK-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i64> [ [[WIDE_MASKED_GATHER]], %[[THEN2]] ], [ zeroinitializer, %[[INNER_HEADER1]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i64> [[VEC_PHI5]], [[VEC_IND]]
-; CHECK-NEXT: [[TMP3]] = add nsw <4 x i64> [[TMP2]], [[VEC_PHI2]]
-; CHECK-NEXT: [[TMP4]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], splat (i64 1000)
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0
-; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[INNER_HEADER1]]
-; CHECK: [[VECTOR_LATCH]]:
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i64.v4p0(<4 x i64> [[TMP3]], <4 x ptr> align 8 [[TMP0]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
-; CHECK: [[OUTER_HEADER]]:
-; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 1000, %[[SCALAR_PH]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
-; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[OUTER_IV]]
-; CHECK-NEXT: store i32 [[C]], ptr [[GEP_A]], align 4
-; CHECK-NEXT: br label %[[INNER_HEADER:.*]]
-; CHECK: [[INNER_HEADER]]:
-; CHECK-NEXT: [[INNER_IV:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[INNER_IV_NEXT:%.*]], %[[INNER_LATCH:.*]] ]
-; CHECK-NEXT: [[RED:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[RED_NEXT:%.*]], %[[INNER_LATCH]] ]
-; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INNER_IV]]
-; CHECK-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[INNER_LATCH]]
-; CHECK: [[THEN]]:
-; CHECK-NEXT: [[L_B:%.*]] = load i64, ptr [[GEP_B]], align 8
-; CHECK-NEXT: br label %[[INNER_LATCH]]
-; CHECK: [[INNER_LATCH]]:
-; CHECK-NEXT: [[P:%.*]] = phi i64 [ 0, %[[INNER_HEADER]] ], [ [[L_B]], %[[THEN]] ]
-; CHECK-NEXT: [[ADD_1:%.*]] = add nsw i64 [[P]], [[OUTER_IV]]
-; CHECK-NEXT: [[RED_NEXT]] = add nsw i64 [[ADD_1]], [[RED]]
-; CHECK-NEXT: [[INNER_IV_NEXT]] = add nuw nsw i64 [[INNER_IV]], 1
-; CHECK-NEXT: [[INNER_EC:%.*]] = icmp eq i64 [[INNER_IV_NEXT]], 1000
-; CHECK-NEXT: br i1 [[INNER_EC]], label %[[OUTER_LATCH]], label %[[INNER_HEADER]]
-; CHECK: [[OUTER_LATCH]]:
-; CHECK-NEXT: [[RED_NEXT_LCSSA:%.*]] = phi i64 [ [[RED_NEXT]], %[[INNER_LATCH]] ]
-; CHECK-NEXT: store i64 [[RED_NEXT_LCSSA]], ptr [[GEP_A]], align 8
-; CHECK-NEXT: [[OUTER_IV_NEXT]] = add nuw nsw i64 [[OUTER_IV]], 1
-; CHECK-NEXT: [[OUTER_EC:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 1000
-; CHECK-NEXT: br i1 [[OUTER_EC]], label %[[EXIT]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: [[EXIT]]:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %outer.header
-
-outer.header: ; preds = %outer.latch, %outer.header.lr.ph
- %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
- %gep.A = getelementptr inbounds i64, ptr %A, i64 %outer.iv
- store i32 %c, ptr %gep.A, align 4
- br label %inner.header
-
-inner.header:
- %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner.latch ]
- %red = phi i64 [ 0, %outer.header ], [ %red.next, %inner.latch ]
- %gep.B = getelementptr inbounds i64, ptr %B, i64 %inner.iv
- br i1 %cond, label %then, label %inner.latch
-
-then:
- %l.b = load i64, ptr %gep.B, align 8
- br label %inner.latch
-
-inner.latch:
- %p = phi i64 [ 0, %inner.header ], [ %l.b, %then ]
- %add.1 = add nsw i64 %p, %outer.iv
- %red.next = add nsw i64 %add.1, %red
- %inner.iv.next = add nuw nsw i64 %inner.iv, 1
- %inner.ec = icmp eq i64 %inner.iv.next, 1000
- br i1 %inner.ec, label %outer.latch, label %inner.header
-
-outer.latch:
- store i64 %red.next, ptr %gep.A, align 8
- %outer.iv.next = add nuw nsw i64 %outer.iv, 1
- %outer.ec = icmp eq i64 %outer.iv.next, 1000
- br i1 %outer.ec, label %exit, label %outer.header, !llvm.loop !1
-
-exit:
- ret void
-}
-
-!1 = distinct !{!1, !2, !3}
-!2 = !{!"llvm.loop.vectorize.width", i32 4}
-!3 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_early_exit.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_early_exit.ll
deleted file mode 100644
index 9dc76c6bf0c43..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/outer_loop_early_exit.ll
+++ /dev/null
@@ -1,49 +0,0 @@
-; REQUIRES: asserts
-; RUN: opt -S -passes=loop-vectorize -enable-vplan-native-path -disable-output -debug 2>&1 < %s | FileCheck %s
-
-; CHECK-LABEL: LV: Found a loop: for.body
-; CHECK: LV: Not vectorizing: Unsupported conditional branch.
-; CHECK: loop not vectorized: loop control flow is not understood by vectorizer
-; CHECK: LV: Not vectorizing: Unsupported outer loop.
-
- at arr2 = external global [8 x i32], align 16
- at arr = external global [8 x [8 x i32]], align 16
-
-define i32 @foo(i32 %n) {
-entry:
- br label %for.body
-
-for.body:
- %iv.outer = phi i64 [ 0, %entry ], [%iv.outer.next, %for.inc ]
- %arrayidx = getelementptr inbounds [8 x i32], ptr @arr2, i64 0, i64 %iv.outer
- %ld1 = load i32, ptr %arrayidx, align 4
- %0 = trunc i64 %iv.outer to i32
- store i32 %0, ptr %arrayidx, align 4
- %1 = trunc i64 %iv.outer to i32
- %add = add nsw i32 %1, %n
- %cmp.early = icmp eq i32 %ld1, 3
- br i1 %cmp.early, label %for.early, label %for.body.inner
-
-for.body.inner:
- %iv.inner = phi i64 [ 0, %for.body ], [ %iv.inner.next, %for.body.inner ]
- %arrayidx7 = getelementptr inbounds [8 x [8 x i32]], ptr @arr, i64 0, i64 %iv.inner, i64 %iv.outer
- store i32 %add, ptr %arrayidx7, align 4
- %iv.inner.next = add nuw nsw i64 %iv.inner, 1
- %cmp.inner = icmp eq i64 %iv.inner.next, 8
- br i1 %cmp.inner, label %for.inc, label %for.body.inner
-
-for.inc:
- %iv.outer.next = add nuw nsw i64 %iv.outer, 1
- %cmp.outer = icmp eq i64%iv.outer.next, 8
- br i1 %cmp.outer, label %for.end, label %for.body, !llvm.loop !1
-
-for.early:
- ret i32 1
-
-for.end:
- ret i32 0
-}
-
-!1 = distinct !{!1, !2, !3}
-!2 = !{!"llvm.loop.vectorize.width", i32 4}
-!3 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_hcfg_construction.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_hcfg_construction.ll
deleted file mode 100644
index 020f84209b23b..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/outer_loop_hcfg_construction.ll
+++ /dev/null
@@ -1,319 +0,0 @@
-; RUN: opt -S -passes=loop-vectorize -enable-vplan-native-path < %s -S | FileCheck %s
-
-; void test(int n, int **a)
-; {
-; for (int k = 0; k < n; ++k) {
-; a[k][0] = 0;
-; #pragma clang loop vectorize_width(4)
-; for (int i = 0; i < n; ++i) {
-; for (int j = 0; j < n; ++j) {
-; a[i][j] = 2 + k;
-; }
-; }
-; }
-; }
-;
-; Make sure VPlan HCFG is constructed when we try to vectorize non-outermost loop
-;
-define void @non_outermost_loop_hcfg_construction(i64 %n, ptr %a) {
-; CHECK-LABEL: define void @non_outermost_loop_hcfg_construction(
-; CHECK-SAME: i64 [[N:%.*]], ptr [[A:%.*]]) {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[OUTERMOST_LOOP:%.*]]
-; CHECK: outermost.loop:
-; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[K_NEXT:%.*]], [[OUTERMOST_LOOP_LATCH:%.*]] ]
-; CHECK-NEXT: [[ARRAYIDX_US:%.*]] = getelementptr inbounds ptr, ptr [[A]], i64 [[K]]
-; CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX_US]], align 8
-; CHECK-NEXT: store i32 0, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[K]] to i32
-; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 2
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP2]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT2]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[MIDDLE_LOOP_LATCH4:%.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[MIDDLE_LOOP_LATCH4]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds ptr, ptr [[A]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x ptr> @llvm.masked.gather.v4p0.v4p0(<4 x ptr> align 8 [[TMP3]], <4 x i1> splat (i1 true), <4 x ptr> poison)
-; CHECK-NEXT: br label [[INNERMOST_LOOP3:%.*]]
-; CHECK: innermost.loop3:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP5:%.*]], [[INNERMOST_LOOP3]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, <4 x ptr> [[WIDE_MASKED_GATHER]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[BROADCAST_SPLAT]], <4 x ptr> align 4 [[TMP4]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP5]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP5]], [[BROADCAST_SPLAT3]]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0
-; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_LOOP_LATCH4]], label [[INNERMOST_LOOP3]]
-; CHECK: vector.latch:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[OUTERMOST_LOOP_LATCH]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[OUTERMOST_LOOP]] ]
-; CHECK-NEXT: br label [[MIDDLE_LOOP:%.*]]
-; CHECK: middle.loop:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[MIDDLE_LOOP_LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT: [[ARRAYIDX11_US_US:%.*]] = getelementptr inbounds ptr, ptr [[A]], i64 [[I]]
-; CHECK-NEXT: [[TMP11:%.*]] = load ptr, ptr [[ARRAYIDX11_US_US]], align 8
-; CHECK-NEXT: br label [[INNERMOST_LOOP:%.*]]
-; CHECK: innermost.loop:
-; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[J_NEXT:%.*]], [[INNERMOST_LOOP]] ], [ 0, [[MIDDLE_LOOP]] ]
-; CHECK-NEXT: [[ARRAYIDX13_US_US:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i64 [[J]]
-; CHECK-NEXT: store i32 [[TMP2]], ptr [[ARRAYIDX13_US_US]], align 4
-; CHECK-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J]], 1
-; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[J_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[MIDDLE_LOOP_LATCH]], label [[INNERMOST_LOOP]]
-; CHECK: middle.loop.latch:
-; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
-; CHECK-NEXT: [[EXITCOND41_NOT:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND41_NOT]], label [[OUTERMOST_LOOP_LATCH]], label [[MIDDLE_LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: outermost.loop.latch:
-; CHECK-NEXT: [[K_NEXT]] = add nuw nsw i64 [[K]], 1
-; CHECK-NEXT: [[EXITCOND47_NOT:%.*]] = icmp eq i64 [[K_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND47_NOT]], label [[OUTERMOST_LOOP_POSTEXIT:%.*]], label [[OUTERMOST_LOOP]]
-; CHECK: outermost.loop.postexit:
-; CHECK-NEXT: br label [[FOR_COND_CLEANUP:%.*]]
-; CHECK: for.cond.cleanup:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %outermost.loop
-
-outermost.loop:
- %k = phi i64 [ 0, %entry ], [ %k.next, %outermost.loop.latch ]
- %arrayidx.us = getelementptr inbounds ptr, ptr %a, i64 %k
- %0 = load ptr, ptr %arrayidx.us, align 8
- store i32 0, ptr %0, align 4
- %1 = trunc i64 %k to i32
- %2 = add i32 %1, 2
- br label %middle.loop
-
-middle.loop:
- %i = phi i64 [ %i.next, %middle.loop.latch ], [ 0, %outermost.loop ]
- %arrayidx11.us.us = getelementptr inbounds ptr, ptr %a, i64 %i
- %3 = load ptr, ptr %arrayidx11.us.us, align 8
- br label %innermost.loop
-
-innermost.loop:
- %j = phi i64 [ %j.next, %innermost.loop ], [ 0, %middle.loop ]
- %arrayidx13.us.us = getelementptr inbounds i32, ptr %3, i64 %j
- store i32 %2, ptr %arrayidx13.us.us, align 4
- %j.next = add nuw nsw i64 %j, 1
- %exitcond.not = icmp eq i64 %j.next, %n
- br i1 %exitcond.not, label %middle.loop.latch, label %innermost.loop
-
-middle.loop.latch:
- %i.next = add nuw nsw i64 %i, 1
- %exitcond41.not = icmp eq i64 %i.next, %n
- br i1 %exitcond41.not, label %outermost.loop.latch, label %middle.loop, !llvm.loop !3
-
-outermost.loop.latch:
- %k.next = add nuw nsw i64 %k, 1
- %exitcond47.not = icmp eq i64 %k.next, %n
- br i1 %exitcond47.not, label %outermost.loop.postexit, label %outermost.loop
-
-outermost.loop.postexit:
- br label %for.cond.cleanup
-
-for.cond.cleanup:
- ret void
-}
-
-; void non_outermost_loop_hcfg_construction_other_loops_at_same_level(long n, int **a)
-; {
-; for (long k = 0; k < n; ++k) {
-; a[k][0] = 0;
-; for (long i = 0; i < n; ++i) {
-; #pragma clang loop vectorize_width(4)
-; for (long j0 = 0; j0 < n; ++j0) {
-; for (long x = 0; x < n; ++x) {
-; a[x+i][j0] = 2 + k+x;
-; }
-; }
-;
-; for (long j1 = n; j1 > 0; --j1) {
-; a[i][j1] *= j1 & 1;
-; }
-; }
-; }
-; }
-;
-; Make sure VPlan HCFG is constructed when we try to vectorize loop with other loops at level > 0
-;
-define void @non_outermost_loop_hcfg_construction_other_loops_at_same_level(i64 %n, ptr %a) {
-; CHECK-LABEL: define void @non_outermost_loop_hcfg_construction_other_loops_at_same_level(
-; CHECK-SAME: i64 [[N:%.*]], ptr [[A:%.*]]) {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[OUTERMOST_LOOP_K:%.*]]
-; CHECK: return:
-; CHECK-NEXT: ret void
-; CHECK: outermost.loop.k:
-; CHECK-NEXT: [[K:%.*]] = phi i64 [ [[K_NEXT:%.*]], [[OUTERMOST_LOOP_K_CLEANUP:%.*]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds ptr, ptr [[A]], i64 [[K]]
-; CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8
-; CHECK-NEXT: store i32 0, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[ADD:%.*]] = add nuw nsw i64 [[K]], 2
-; CHECK-NEXT: br label [[MIDDLE_LOOP_I:%.*]]
-; CHECK: middle.loop.i:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, [[OUTERMOST_LOOP_K]] ], [ [[I_NEXT:%.*]], [[MIDDLE_LOOP_I_CLEANUP:%.*]] ]
-; CHECK-NEXT: [[INVARIANT_GEP:%.*]] = getelementptr ptr, ptr [[A]], i64 [[I]]
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[ADD]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x i64> poison, i64 [[N]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT2]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[MIDDLE_LOOP_J0_CLEANUP4:%.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[MIDDLE_LOOP_J0_CLEANUP4]] ]
-; CHECK-NEXT: br label [[INNERMOST_LOOP3:%.*]]
-; CHECK: innermost.loop3:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP5:%.*]], [[INNERMOST_LOOP3]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw <4 x i64> [[BROADCAST_SPLAT]], [[VEC_PHI]]
-; CHECK-NEXT: [[TMP2:%.*]] = trunc <4 x i64> [[TMP1]] to <4 x i32>
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr ptr, ptr [[INVARIANT_GEP]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x ptr> @llvm.masked.gather.v4p0.v4p0(<4 x ptr> align 8 [[TMP3]], <4 x i1> splat (i1 true), <4 x ptr> poison)
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, <4 x ptr> [[WIDE_MASKED_GATHER]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[TMP2]], <4 x ptr> align 4 [[TMP4]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP5]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP5]], [[BROADCAST_SPLAT3]]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0
-; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_LOOP_J0_CLEANUP4]], label [[INNERMOST_LOOP3]]
-; CHECK: vector.latch:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[INNERMOST_LOOP_J1_LR_PH:%.*]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[MIDDLE_LOOP_I]] ]
-; CHECK-NEXT: br label [[MIDDLE_LOOP_J0_PH:%.*]]
-; CHECK: outermost.loop.k.cleanup:
-; CHECK-NEXT: [[K_NEXT]] = add nuw nsw i64 [[K]], 1
-; CHECK-NEXT: [[EXITCOND71_NOT:%.*]] = icmp eq i64 [[K_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND71_NOT]], label [[RETURN:%.*]], label [[OUTERMOST_LOOP_K]]
-; CHECK: innermost.loop.j1.lr.ph:
-; CHECK-NEXT: [[TMP11:%.*]] = load ptr, ptr [[INVARIANT_GEP]], align 8
-; CHECK-NEXT: br label [[INNERMOST_LOOP_J1:%.*]]
-; CHECK: middle.loop.j0.ph:
-; CHECK-NEXT: [[J0:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[J0_NEXT:%.*]], [[MIDDLE_LOOP_J0_CLEANUP:%.*]] ]
-; CHECK-NEXT: br label [[INNERMOST_LOOP:%.*]]
-; CHECK: middle.loop.j0.cleanup:
-; CHECK-NEXT: [[J0_NEXT]] = add nuw nsw i64 [[J0]], 1
-; CHECK-NEXT: [[J0_EXIT_COND_NOT:%.*]] = icmp eq i64 [[J0_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[J0_EXIT_COND_NOT]], label [[INNERMOST_LOOP_J1_LR_PH]], label [[MIDDLE_LOOP_J0_PH]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK: innermost.loop:
-; CHECK-NEXT: [[X:%.*]] = phi i64 [ 0, [[MIDDLE_LOOP_J0_PH]] ], [ [[X_NEXT:%.*]], [[INNERMOST_LOOP]] ]
-; CHECK-NEXT: [[ADD14:%.*]] = add nuw nsw i64 [[ADD]], [[X]]
-; CHECK-NEXT: [[CONV:%.*]] = trunc i64 [[ADD14]] to i32
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr ptr, ptr [[INVARIANT_GEP]], i64 [[X]]
-; CHECK-NEXT: [[TMP12:%.*]] = load ptr, ptr [[GEP]], align 8
-; CHECK-NEXT: [[ARRAYIDX17:%.*]] = getelementptr inbounds i32, ptr [[TMP12]], i64 [[J0]]
-; CHECK-NEXT: store i32 [[CONV]], ptr [[ARRAYIDX17]], align 4
-; CHECK-NEXT: [[X_NEXT]] = add nuw nsw i64 [[X]], 1
-; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[X_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[MIDDLE_LOOP_J0_CLEANUP]], label [[INNERMOST_LOOP]]
-; CHECK: middle.loop.i.cleanup:
-; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
-; CHECK-NEXT: [[EXITCOND70_NOT:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
-; CHECK-NEXT: br i1 [[EXITCOND70_NOT]], label [[OUTERMOST_LOOP_K_CLEANUP]], label [[MIDDLE_LOOP_I]]
-; CHECK: innermost.loop.j1:
-; CHECK-NEXT: [[J21_064:%.*]] = phi i64 [ [[N]], [[INNERMOST_LOOP_J1_LR_PH]] ], [ [[DEC:%.*]], [[INNERMOST_LOOP_J1]] ]
-; CHECK-NEXT: [[ARRAYIDX28:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i64 [[J21_064]]
-; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX28]], align 4
-; CHECK-NEXT: [[TMP14:%.*]] = and i64 [[J21_064]], 1
-; CHECK-NEXT: [[DOTNOT:%.*]] = icmp eq i64 [[TMP14]], 0
-; CHECK-NEXT: [[CONV30:%.*]] = select i1 [[DOTNOT]], i32 0, i32 [[TMP13]]
-; CHECK-NEXT: store i32 [[CONV30]], ptr [[ARRAYIDX28]], align 4
-; CHECK-NEXT: [[DEC]] = add nsw i64 [[J21_064]], -1
-; CHECK-NEXT: [[CMP23:%.*]] = icmp sgt i64 [[J21_064]], 1
-; CHECK-NEXT: br i1 [[CMP23]], label [[INNERMOST_LOOP_J1]], label [[MIDDLE_LOOP_I_CLEANUP]]
-;
-entry:
- br label %outermost.loop.k
-
-return:
- ret void
-
-outermost.loop.k:
- %k = phi i64 [ %k.next, %outermost.loop.k.cleanup ], [ 0, %entry ]
- %arrayidx = getelementptr inbounds ptr, ptr %a, i64 %k
- %0 = load ptr, ptr %arrayidx, align 8
- store i32 0, ptr %0, align 4
- %add = add nuw nsw i64 %k, 2
- br label %middle.loop.i
-
-middle.loop.i:
- %i = phi i64 [ 0, %outermost.loop.k ], [ %i.next, %middle.loop.i.cleanup ]
- %invariant.gep = getelementptr ptr, ptr %a, i64 %i
- br label %middle.loop.j0.ph
-
-outermost.loop.k.cleanup:
- %k.next = add nuw nsw i64 %k, 1
- %exitcond71.not = icmp eq i64 %k.next, %n
- br i1 %exitcond71.not, label %return, label %outermost.loop.k
-
-innermost.loop.j1.lr.ph: ; preds = %middle.loop.j0.cleanup
- %1 = load ptr, ptr %invariant.gep, align 8
- br label %innermost.loop.j1
-
-middle.loop.j0.ph:
- %j0 = phi i64 [ 0, %middle.loop.i ], [ %j0.next, %middle.loop.j0.cleanup ]
- br label %innermost.loop
-
-middle.loop.j0.cleanup:
- %j0.next = add nuw nsw i64 %j0, 1
- %j0.exit.cond.not = icmp eq i64 %j0.next, %n
- br i1 %j0.exit.cond.not, label %innermost.loop.j1.lr.ph, label %middle.loop.j0.ph, !llvm.loop !3
-
-innermost.loop:
- %x = phi i64 [ 0, %middle.loop.j0.ph ], [ %x.next, %innermost.loop ]
- %add14 = add nuw nsw i64 %add, %x
- %conv = trunc i64 %add14 to i32
- %gep = getelementptr ptr, ptr %invariant.gep, i64 %x
- %2 = load ptr, ptr %gep, align 8
- %arrayidx17 = getelementptr inbounds i32, ptr %2, i64 %j0
- store i32 %conv, ptr %arrayidx17, align 4
- %x.next = add nuw nsw i64 %x, 1
- %exitcond.not = icmp eq i64 %x.next, %n
- br i1 %exitcond.not, label %middle.loop.j0.cleanup, label %innermost.loop
-
-middle.loop.i.cleanup:
- %i.next = add nuw nsw i64 %i, 1
- %exitcond70.not = icmp eq i64 %i.next, %n
- br i1 %exitcond70.not, label %outermost.loop.k.cleanup, label %middle.loop.i
-
-innermost.loop.j1:
- %j21.064 = phi i64 [ %n, %innermost.loop.j1.lr.ph ], [ %dec, %innermost.loop.j1 ]
- %arrayidx28 = getelementptr inbounds i32, ptr %1, i64 %j21.064
- %3 = load i32, ptr %arrayidx28, align 4
- %4 = and i64 %j21.064, 1
- %.not = icmp eq i64 %4, 0
- %conv30 = select i1 %.not, i32 0, i32 %3
- store i32 %conv30, ptr %arrayidx28, align 4
- %dec = add nsw i64 %j21.064, -1
- %cmp23 = icmp sgt i64 %j21.064, 1
- br i1 %cmp23, label %innermost.loop.j1, label %middle.loop.i.cleanup
-}
-
-!3 = distinct !{!3, !4, !5, !6}
-!4 = !{!"llvm.loop.vectorize.width", i32 4}
-!5 = !{!"llvm.loop.vectorize.scalable.enable", i1 false}
-!6 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll
deleted file mode 100644
index 7653d98988975..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll
+++ /dev/null
@@ -1,140 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
-; RUN: opt -S -pass-remarks=loop-vectorize -passes=loop-vectorize -enable-vplan-native-path -force-target-supports-scalable-vectors < %s 2>&1 | FileCheck %s
-; RUN: opt -S -pass-remarks=loop-vectorize -passes=loop-vectorize -enable-vplan-native-path < %s 2>&1 | FileCheck %s -check-prefix=NO_SCALABLE_VECS
-
-; Test if the vplan-native-path successfully vectorizes a loop using scalable
-; vectors if the target supports scalable vectors and rejects vectorization
-; if a scalable VF is requested but not supported by the target.
-
-; CHECK: remark: <unknown>:0:0: vectorized outer loop (vectorization width: vscale x 4, interleaved count: 1)
-; NO_SCALABLE_VECS: remark: <unknown>:0:0: loop not vectorized: the scalable user-specified vectorization width for outer-loop vectorization cannot be used because the target does not support scalable vectors.
-
- at A = external global [1024 x float], align 4
- at B = external global [512 x float], align 4
-
-define void @foo() {
-; CHECK-LABEL: define void @foo() {
-; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
-; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
-; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0
-; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[OUTER_LOOP_LATCH4:%.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[OUTER_LOOP_LATCH4]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [1024 x float], ptr @A, i64 0, <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP10]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> poison)
-; CHECK-NEXT: br label [[INNER_LOOP1:%.*]]
-; CHECK: inner_loop1:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP13:%.*]], [[INNER_LOOP1]] ]
-; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 4 x float> [ [[WIDE_MASKED_GATHER]], [[VECTOR_BODY]] ], [ [[TMP12:%.*]], [[INNER_LOOP1]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [512 x float], ptr @B, i64 0, <vscale x 4 x i64> [[VEC_PHI]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> poison)
-; CHECK-NEXT: [[TMP12]] = fmul <vscale x 4 x float> [[VEC_PHI2]], [[WIDE_MASKED_GATHER3]]
-; CHECK-NEXT: [[TMP13]] = add nuw nsw <vscale x 4 x i64> [[VEC_PHI]], splat (i64 1)
-; CHECK-NEXT: [[TMP14:%.*]] = icmp eq <vscale x 4 x i64> [[TMP13]], splat (i64 512)
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <vscale x 4 x i1> [[TMP14]], i32 0
-; CHECK-NEXT: br i1 [[TMP15]], label [[OUTER_LOOP_LATCH4]], label [[INNER_LOOP1]]
-; CHECK: vector.latch:
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> [[TMP12]], <vscale x 4 x ptr> align 4 [[TMP10]], <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK: scalar.ph:
-; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT: br label [[OUTER_LOOP:%.*]]
-; CHECK: outer_loop:
-; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[OUTER_LOOP_LATCH:%.*]] ]
-; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 [[I]]
-; CHECK-NEXT: [[X_START:%.*]] = load float, ptr [[ARRAYIDX1]], align 4
-; CHECK-NEXT: br label [[INNER_LOOP:%.*]]
-; CHECK: inner_loop:
-; CHECK-NEXT: [[J:%.*]] = phi i64 [ 0, [[OUTER_LOOP]] ], [ [[J_NEXT:%.*]], [[INNER_LOOP]] ]
-; CHECK-NEXT: [[X:%.*]] = phi float [ [[X_START]], [[OUTER_LOOP]] ], [ [[X_NEXT:%.*]], [[INNER_LOOP]] ]
-; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [512 x float], ptr @B, i64 0, i64 [[J]]
-; CHECK-NEXT: [[B:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
-; CHECK-NEXT: [[X_NEXT]] = fmul float [[X]], [[B]]
-; CHECK-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J]], 1
-; CHECK-NEXT: [[INNER_EXITCOND:%.*]] = icmp eq i64 [[J_NEXT]], 512
-; CHECK-NEXT: br i1 [[INNER_EXITCOND]], label [[OUTER_LOOP_LATCH]], label [[INNER_LOOP]]
-; CHECK: outer_loop_latch:
-; CHECK-NEXT: [[X_NEXT_LCSSA:%.*]] = phi float [ [[X_NEXT]], [[INNER_LOOP]] ]
-; CHECK-NEXT: store float [[X_NEXT_LCSSA]], ptr [[ARRAYIDX1]], align 4
-; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
-; CHECK-NEXT: [[OUTER_EXITCOND:%.*]] = icmp eq i64 [[I_NEXT]], 1024
-; CHECK-NEXT: br i1 [[OUTER_EXITCOND]], label [[EXIT]], label [[OUTER_LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK: exit:
-; CHECK-NEXT: ret void
-;
-; NO_SCALABLE_VECS-LABEL: define void @foo() {
-; NO_SCALABLE_VECS-NEXT: entry:
-; NO_SCALABLE_VECS-NEXT: br label [[OUTER_LOOP:%.*]]
-; NO_SCALABLE_VECS: outer_loop:
-; NO_SCALABLE_VECS-NEXT: [[I:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[OUTER_LOOP_LATCH:%.*]] ]
-; NO_SCALABLE_VECS-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 [[I]]
-; NO_SCALABLE_VECS-NEXT: [[X_START:%.*]] = load float, ptr [[ARRAYIDX1]], align 4
-; NO_SCALABLE_VECS-NEXT: br label [[INNER_LOOP:%.*]]
-; NO_SCALABLE_VECS: inner_loop:
-; NO_SCALABLE_VECS-NEXT: [[J:%.*]] = phi i64 [ 0, [[OUTER_LOOP]] ], [ [[J_NEXT:%.*]], [[INNER_LOOP]] ]
-; NO_SCALABLE_VECS-NEXT: [[X:%.*]] = phi float [ [[X_START]], [[OUTER_LOOP]] ], [ [[X_NEXT:%.*]], [[INNER_LOOP]] ]
-; NO_SCALABLE_VECS-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [512 x float], ptr @B, i64 0, i64 [[J]]
-; NO_SCALABLE_VECS-NEXT: [[B:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
-; NO_SCALABLE_VECS-NEXT: [[X_NEXT]] = fmul float [[X]], [[B]]
-; NO_SCALABLE_VECS-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J]], 1
-; NO_SCALABLE_VECS-NEXT: [[INNER_EXITCOND:%.*]] = icmp eq i64 [[J_NEXT]], 512
-; NO_SCALABLE_VECS-NEXT: br i1 [[INNER_EXITCOND]], label [[OUTER_LOOP_LATCH]], label [[INNER_LOOP]]
-; NO_SCALABLE_VECS: outer_loop_latch:
-; NO_SCALABLE_VECS-NEXT: [[X_NEXT_LCSSA:%.*]] = phi float [ [[X_NEXT]], [[INNER_LOOP]] ]
-; NO_SCALABLE_VECS-NEXT: store float [[X_NEXT_LCSSA]], ptr [[ARRAYIDX1]], align 4
-; NO_SCALABLE_VECS-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
-; NO_SCALABLE_VECS-NEXT: [[OUTER_EXITCOND:%.*]] = icmp eq i64 [[I_NEXT]], 1024
-; NO_SCALABLE_VECS-NEXT: br i1 [[OUTER_EXITCOND]], label [[EXIT:%.*]], label [[OUTER_LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
-; NO_SCALABLE_VECS: exit:
-; NO_SCALABLE_VECS-NEXT: ret void
-;
-entry:
- br label %outer_loop
-
-outer_loop:
- %i = phi i64 [ 0, %entry ], [ %i.next, %outer_loop_latch ]
- %arrayidx1 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %i
- %x.start = load float, ptr %arrayidx1, align 4
- br label %inner_loop
-
-inner_loop:
- %j = phi i64 [ 0, %outer_loop ], [ %j.next, %inner_loop ]
- %x = phi float [ %x.start, %outer_loop ], [ %x.next, %inner_loop ]
- %arrayidx2 = getelementptr inbounds [512 x float], ptr @B, i64 0, i64 %j
- %b = load float, ptr %arrayidx2, align 4
- %x.next = fmul float %x, %b
- %j.next = add nuw nsw i64 %j, 1
- %inner_exitcond = icmp eq i64 %j.next, 512
- br i1 %inner_exitcond, label %outer_loop_latch, label %inner_loop
-
-outer_loop_latch:
- store float %x.next, ptr %arrayidx1, align 4
- %i.next = add nuw nsw i64 %i, 1
- %outer_exitcond = icmp eq i64 %i.next, 1024
- br i1 %outer_exitcond, label %exit, label %outer_loop, !llvm.loop !1
-
-exit:
- ret void
-}
-
-!1 = distinct !{!1, !2, !3, !4}
-!2 = !{!"llvm.loop.vectorize.enable", i1 true}
-!3 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
-!4 = !{!"llvm.loop.vectorize.width", i32 4}
diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll
deleted file mode 100644
index 275fb9015a95f..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/outer_loop_test1.ll
+++ /dev/null
@@ -1,88 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 5
-; RUN: opt -S -passes=loop-vectorize -enable-vplan-native-path -verify-loop-info -verify-dom-info < %s | FileCheck %s
-
-; extern int arr[8][8];
-; extern int arr2[8];
-;
-; void foo(int n)
-; {
-; int i1, i2;
-;
-; #pragma clang loop vectorize(enable) vectorize_width(4)
-; for (i1 = 0; i1 < 8; i1++) {
-; arr2[i1] = i1;
-; for (i2 = 0; i2 < 8; i2++)
-; arr[i2][i1] = i1 + n;
-; }
-; }
-;
-
- at arr2 = external global [8 x i32], align 16
- at arr = external global [8 x [8 x i32]], align 16
-
-define void @foo(i32 %n) {
-; CHECK-LABEL: define void @foo(
-; CHECK-SAME: i32 [[N:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[N]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [8 x i32], ptr @arr2, i64 0, <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[TMP1:%.*]] = trunc <4 x i64> [[VEC_IND]] to <4 x i32>
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[TMP1]], <4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT: br label %[[FOR_BODY31:.*]]
-; CHECK: [[FOR_BODY31]]:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[FOR_BODY31]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [8 x [8 x i32]], ptr @arr, i64 0, <4 x i64> [[VEC_PHI]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> [[TMP2]], <4 x ptr> align 4 [[TMP3]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP4]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], splat (i64 8)
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0
-; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[FOR_BODY31]]
-; CHECK: [[VECTOR_LATCH]]:
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br i1 true, [[FOR_END10:label %.*]], label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
-;
-entry:
- br label %for.body
-
-for.body: ; preds = %for.inc8, %entry
- %indvars.iv21 = phi i64 [ 0, %entry ], [ %indvars.iv.next22, %for.inc8 ]
- %arrayidx = getelementptr inbounds [8 x i32], ptr @arr2, i64 0, i64 %indvars.iv21
- %0 = trunc i64 %indvars.iv21 to i32
- store i32 %0, ptr %arrayidx, align 4
- %1 = trunc i64 %indvars.iv21 to i32
- %add = add nsw i32 %1, %n
- br label %for.body3
-
-for.body3: ; preds = %for.body3, %for.body
- %indvars.iv = phi i64 [ 0, %for.body ], [ %indvars.iv.next, %for.body3 ]
- %arrayidx7 = getelementptr inbounds [8 x [8 x i32]], ptr @arr, i64 0, i64 %indvars.iv, i64 %indvars.iv21
- store i32 %add, ptr %arrayidx7, align 4
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %exitcond = icmp eq i64 %indvars.iv.next, 8
- br i1 %exitcond, label %for.inc8, label %for.body3
-
-for.inc8: ; preds = %for.body3
- %indvars.iv.next22 = add nuw nsw i64 %indvars.iv21, 1
- %exitcond23 = icmp eq i64 %indvars.iv.next22, 8
- br i1 %exitcond23, label %for.end10, label %for.body, !llvm.loop !1
-
-for.end10: ; preds = %for.inc8
- ret void
-}
-
-!1 = distinct !{!1, !2, !3}
-!2 = !{!"llvm.loop.vectorize.width", i32 4}
-!3 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_test2.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_test2.ll
deleted file mode 100644
index c6fb9230740b3..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/outer_loop_test2.ll
+++ /dev/null
@@ -1,107 +0,0 @@
-; int A[1024], B[1024];
-;
-; void foo(int iCount, int c, int jCount)
-; {
-;
-; int i, j;
-;
-; #pragma clang loop vectorize(enable) vectorize_width(4)
-; for (i = 0; i < iCount; i++) {
-; A[i] = c;
-; for (j = 0; j < jCount; j++) {
-; A[i] += B[j] + i;
-; }
-; }
-; }
-; RUN: opt -S -passes=loop-vectorize -enable-vplan-native-path < %s | FileCheck %s
-; CHECK: %[[ZeroTripChk:.*]] = icmp sgt i32 %jCount, 0
-; CHECK-LABEL: vector.ph:
-; CHECK: %[[CVal0:.*]] = insertelement <4 x i32> poison, i32 %c, i64 0
-; CHECK-NEXT: %[[CSplat:.*]] = shufflevector <4 x i32> %[[CVal0]], <4 x i32> poison, <4 x i32> zeroinitializer
-
-; CHECK-LABEL: vector.body:
-; CHECK: %[[Ind:.*]] = phi i64 [ 0, %vector.ph ], [ %[[IndNext:.*]], %vector.latch ]
-; CHECK: %[[VecInd:.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %vector.ph ], [ %[[VecIndNext:.*]], %vector.latch ]
-; CHECK: %[[AAddr:.*]] = getelementptr inbounds [1024 x i32], ptr @A, i64 0, <4 x i64> %[[VecInd]]
-; CHECK: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %[[CSplat]], <4 x ptr> align 4 %[[AAddr]], <4 x i1> splat (i1 true))
-; CHECK: br i1 %[[ZeroTripChk]], label %[[InnerForPh:.*]], label %vector.latch
-
-; CHECK: [[InnerForPh]]:
-; CHECK: %[[WideAVal:.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 %[[AAddr]], <4 x i1> splat (i1 true), <4 x i32> poison)
-; CHECK: %[[VecIndTr:.*]] = trunc <4 x i64> %[[VecInd]] to <4 x i32>
-; CHECK: br label %[[InnerForBody:.*]]
-
-; CHECK: [[InnerForBody]]:
-; CHECK: %[[InnerInd:.*]] = phi <4 x i64> [ zeroinitializer, %[[InnerForPh]] ], [ %[[InnerIndNext:.*]], %[[InnerForBody]] ]
-; CHECK: %[[AccumPhi:.*]] = phi <4 x i32> [ %[[WideAVal]], %[[InnerForPh]] ], [ %[[AccumPhiNext:.*]], %[[InnerForBody]] ]
-; CHECK: %[[BAddr:.*]] = getelementptr inbounds [1024 x i32], ptr @B, i64 0, <4 x i64> %[[InnerInd]]
-; CHECK: %[[WideBVal:.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 %[[BAddr]], <4 x i1> splat (i1 true), <4 x i32> poison)
-; CHECK: %[[Add1:.*]] = add nsw <4 x i32> %[[WideBVal]], %[[VecIndTr]]
-; CHECK: %[[AccumPhiNext]] = add nsw <4 x i32> %[[Add1]], %[[AccumPhi]]
-; CHECK: %[[InnerIndNext]] = add nuw nsw <4 x i64> %[[InnerInd]], splat (i64 1)
-; CHECK: %[[InnerVecCond:.*]] = icmp eq <4 x i64> %[[InnerIndNext]], {{.*}}
-; CHECK: %[[InnerCond:.+]] = extractelement <4 x i1> %[[InnerVecCond]], i32 0
-; CHECK: br i1 %[[InnerCond]], label %[[InnerCrit:.*]], label %[[InnerForBody]]
-
-; CHECK: [[InnerCrit]]:
-; CHECK: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %[[AccumPhiNext]], <4 x ptr> align 4 %[[AAddr]], <4 x i1> splat (i1 true))
-; CHECK: br label %vector.latch
-
-; CHECK: vector.latch:
-; CHECK: %[[IndNext]] = add nuw i64 %[[Ind]], 4
-; CHECK: %[[VecIndNext]] = add nuw nsw <4 x i64> %[[VecInd]], splat (i64 4)
-; CHECK: %[[Cmp:.*]] = icmp eq i64 %[[IndNext]], {{.*}}
-; CHECK: br i1 %[[Cmp]], label %middle.block, label %vector.body
-
- at A = common global [1024 x i32] zeroinitializer, align 16
- at B = common global [1024 x i32] zeroinitializer, align 16
-
-define void @foo(i32 %iCount, i32 %c, i32 %jCount) {
-entry:
- %cmp22 = icmp sgt i32 %iCount, 0
- br i1 %cmp22, label %for.body.lr.ph, label %for.end11
-
-for.body.lr.ph: ; preds = %entry
- %cmp220 = icmp sgt i32 %jCount, 0
- %wide.trip.count = zext i32 %jCount to i64
- %wide.trip.count27 = zext i32 %iCount to i64
- br label %for.body
-
-for.body: ; preds = %for.inc9, %for.body.lr.ph
- %indvars.iv25 = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next26, %for.inc9 ]
- %arrayidx = getelementptr inbounds [1024 x i32], ptr @A, i64 0, i64 %indvars.iv25
- store i32 %c, ptr %arrayidx, align 4
- br i1 %cmp220, label %for.body3.lr.ph, label %for.inc9
-
-for.body3.lr.ph: ; preds = %for.body
- %arrayidx.promoted = load i32, ptr %arrayidx, align 4
- %0 = trunc i64 %indvars.iv25 to i32
- br label %for.body3
-
-for.body3: ; preds = %for.body3, %for.body3.lr.ph
- %indvars.iv = phi i64 [ 0, %for.body3.lr.ph ], [ %indvars.iv.next, %for.body3 ]
- %1 = phi i32 [ %arrayidx.promoted, %for.body3.lr.ph ], [ %add8, %for.body3 ]
- %arrayidx5 = getelementptr inbounds [1024 x i32], ptr @B, i64 0, i64 %indvars.iv
- %2 = load i32, ptr %arrayidx5, align 4
- %add = add nsw i32 %2, %0
- %add8 = add nsw i32 %add, %1
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
- br i1 %exitcond, label %for.cond1.for.inc9_crit_edge, label %for.body3
-
-for.cond1.for.inc9_crit_edge: ; preds = %for.body3
- store i32 %add8, ptr %arrayidx, align 4
- br label %for.inc9
-
-for.inc9: ; preds = %for.cond1.for.inc9_crit_edge, %for.body
- %indvars.iv.next26 = add nuw nsw i64 %indvars.iv25, 1
- %exitcond28 = icmp eq i64 %indvars.iv.next26, %wide.trip.count27
- br i1 %exitcond28, label %for.end11, label %for.body, !llvm.loop !1
-
-for.end11: ; preds = %for.inc9, %entry
- ret void
-}
-
-!1 = distinct !{!1, !2, !3}
-!2 = !{!"llvm.loop.vectorize.width", i32 4}
-!3 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-outer-loop-uncomputable-trip-count.ll b/llvm/test/Transforms/LoopVectorize/vplan-outer-loop-uncomputable-trip-count.ll
deleted file mode 100644
index fabcb663bb7d4..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/vplan-outer-loop-uncomputable-trip-count.ll
+++ /dev/null
@@ -1,47 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -S %s | FileCheck %s
-
-declare i1 @cond()
-
-; Make sure we do not vectorize (or crash) on outer loops with uncomputable
-; trip-counts.
-
-define void @test() {
-; CHECK-LABEL: @test(
-; CHECK-NEXT: entry:
-; CHECK-NEXT: br label [[FOR1_HEADER:%.*]]
-; CHECK: for1.header:
-; CHECK-NEXT: br label [[FOR2_HEADER:%.*]]
-; CHECK: for2.header:
-; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR1_HEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR2_HEADER]] ]
-; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 0
-; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR1_LATCH:%.*]], label [[FOR2_HEADER]]
-; CHECK: for1.latch:
-; CHECK-NEXT: [[C:%.*]] = call i1 @cond()
-; CHECK-NEXT: br i1 [[C]], label [[EXIT:%.*]], label [[FOR1_HEADER]], !llvm.loop !0
-; CHECK: exit:
-; CHECK-NEXT: ret void
-;
-entry:
- br label %for1.header
-
-for1.header: ; preds = %for.cond.cleanup3, %for.cond1.preheader.lr.ph
- br label %for2.header
-
-for2.header: ; preds = %for.body10, %for.body10.preheader
- %indvars.iv = phi i64 [ 0, %for1.header ], [ %indvars.iv.next, %for2.header ]
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %exitcond = icmp eq i64 %indvars.iv.next, 0
- br i1 %exitcond, label %for1.latch, label %for2.header
-
-for1.latch: ; preds = %for.cond.cleanup9
- %c = call i1 @cond()
- br i1 %c, label %exit, label %for1.header, !llvm.loop !0
-
-exit: ; preds = %for.cond.cleanup3, %entry
- ret void
-}
-
-!0 = distinct !{!0, !1}
-!1 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll b/llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll
deleted file mode 100644
index 97847a6e1526f..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll
+++ /dev/null
@@ -1,78 +0,0 @@
-; RUN: opt -passes=loop-vectorize -force-vector-width=4 -enable-vplan-native-path -S %s | FileCheck %s
-
-; Vectorize explict marked outer loop using vplan native path. Inner loop
-; contains simple double add reduction. IR is compiled and modified by hand
-; from following C code:
-; void inner_loop_reduction(const ptr restrict in_a, const ptr restrict in_b, ptr restrict out)
-; {
-; #pragma clang loop vectorize(enable)
-; for (int i = 0; i < 1000; ++i) {
-; double a = in_a[i];
-; double b = in_b[i];
-; for (int j = 0; j < 10000; ++j) {
-; a = a + b;
-; }
-; out[i] = a;
-; }
-; }
-define void @inner_loop_reduction(ptr noalias nocapture readonly %a.in, ptr noalias nocapture readonly %b.in, ptr noalias nocapture %c.out) {
-; CHECK-LABEL: @inner_loop_reduction(
-
-; CHECK: vector.body:
-; CHECK-NEXT: %[[FOR1_INDEX:.*]] = phi i64 [ 0, %[[LABEL_PR:.*]] ], [ %{{.*}}, %[[LABEL_FOR1_LATCH:.*]] ]
-; CHECK: %[[VEC_INDEX:.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[LABEL_PR]] ], [ %{{.*}}, %[[LABEL_FOR1_LATCH]] ]
-; CHECK-NEXT: %[[A_PTR:.*]] = getelementptr inbounds double, ptr %a.in, <4 x i64> %[[VEC_INDEX]]
-; CHECK-NEXT: %[[MASKED_GATHER1:.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> align 8 %[[A_PTR]], <4 x i1> splat (i1 true), <4 x double> poison)
-; CHECK-NEXT: %[[B_PTR:.*]] = getelementptr inbounds double, ptr %b.in, <4 x i64> %[[VEC_INDEX]]
-; CHECK-NEXT: %[[MASKED_GATHER2:.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> align 8 %[[B_PTR]], <4 x i1> splat (i1 true), <4 x double> poison)
-; CHECK-NEXT: br label %[[FOR2_HEADER:.*]]
-
-; CHECK: [[FOR2_HEADER]]:
-; CHECK-NEXT: %[[FOR2_INDEX:.*]] = phi <4 x i32> [ zeroinitializer, %vector.body ], [ %[[FOR2_INDEX_NEXT:.*]], %[[FOR2_HEADER]] ]
-; CHECK-NEXT: %[[REDUCTION:.*]] = phi <4 x double> [ %[[MASKED_GATHER1]], %vector.body ], [ %[[REDUCTION_NEXT:.*]], %[[FOR2_HEADER]] ]
-; CHECK-NEXT: %[[REDUCTION_NEXT]] = fadd <4 x double> %[[MASKED_GATHER2]], %[[REDUCTION]]
-; CHECK-NEXT: %[[FOR2_INDEX_NEXT]] = add nuw nsw <4 x i32> %[[FOR2_INDEX]], splat (i32 1)
-; CHECK-NEXT: %[[VEC_PTR:.*]] = icmp eq <4 x i32> %[[FOR2_INDEX_NEXT]], splat (i32 10000)
-; CHECK-NEXT: %[[EXIT_COND:.*]] = extractelement <4 x i1> %[[VEC_PTR]], i32 0
-; CHECK-NEXT: br i1 %[[EXIT_COND]], label %[[FOR1_LATCH:.*]], label %{{.*}}
-
-; CHECK: [[FOR1_LATCH]]:
-; CHECK-NEXT: %[[C_PTR:.*]] = getelementptr inbounds double, ptr %c.out, <4 x i64> %[[VEC_INDEX]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> %[[REDUCTION_NEXT]], <4 x ptr> align 8 %[[C_PTR]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: %[[FOR1_INDEX_NEXT:.*]] = add nuw i64 %[[FOR1_INDEX]], 4
-; CHECK-NEXT: %{{.*}} = add nuw nsw <4 x i64> %[[VEC_INDEX]], splat (i64 4)
-; CHECK-NEXT: %[[EXIT_COND:.*]] = icmp eq i64 %[[FOR1_INDEX_NEXT]], 1000
-; CHECK-NEXT: br i1 %[[EXIT_COND]], label %{{.*}}, label %vector.body
-
-entry:
- br label %for1.header
-
-for1.header: ; preds = %entry
- %indvar1 = phi i64 [ 0, %entry ], [ %indvar11, %for1.latch ]
- %a.ptr = getelementptr inbounds double, ptr %a.in, i64 %indvar1
- %a = load double, ptr %a.ptr, align 8
- %b.ptr = getelementptr inbounds double, ptr %b.in, i64 %indvar1
- %b = load double, ptr %b.ptr, align 8
- br label %for2.header
-
-for2.header: ; preds = %for1.header, %for2.header
- %indvar2 = phi i32 [ 0, %for1.header ], [ %indvar21, %for2.header ]
- %a.reduction = phi double [ %a, %for1.header ], [ %a.reduction1, %for2.header ]
- %a.reduction1 = fadd double %b, %a.reduction
- %indvar21 = add nuw nsw i32 %indvar2, 1
- %for2.cond = icmp eq i32 %indvar21, 10000
- br i1 %for2.cond, label %for1.latch, label %for2.header
-
-for1.latch: ; preds = %for2.header
- %c.ptr = getelementptr inbounds double, ptr %c.out, i64 %indvar1
- store double %a.reduction1, ptr %c.ptr, align 8
- %indvar11 = add nuw nsw i64 %indvar1, 1
- %for1.cond = icmp eq i64 %indvar11, 1000
- br i1 %for1.cond, label %exit, label %for1.header, !llvm.loop !0
-
-exit: ; preds = %for1.latch
- ret void
-}
-
-!0 = distinct !{!0, !1}
-!1 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-widen-call-instruction.ll b/llvm/test/Transforms/LoopVectorize/widen-call.ll
similarity index 56%
rename from llvm/test/Transforms/LoopVectorize/vplan-widen-call-instruction.ll
rename to llvm/test/Transforms/LoopVectorize/widen-call.ll
index 46f50c18299b9..559a1b3f8ec71 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-widen-call-instruction.ll
+++ b/llvm/test/Transforms/LoopVectorize/widen-call.ll
@@ -1,47 +1,14 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -passes=loop-vectorize -force-vector-width=4 -enable-vplan-native-path -S %s | FileCheck %s
-
-; Test that VPlan native path is able to widen call intructions like
-; llvm.sqrt.* intrincis calls.
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
declare double @llvm.sqrt.f64(double %0)
define void @widen_call_instruction(ptr noalias nocapture readonly %a.in, ptr noalias nocapture readonly %b.in, ptr noalias nocapture %c.out) {
; CHECK-LABEL: define void @widen_call_instruction(
; CHECK-SAME: ptr noalias readonly captures(none) [[A_IN:%.*]], ptr noalias readonly captures(none) [[B_IN:%.*]], ptr noalias captures(none) [[C_OUT:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr [[A_IN]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> align 8 [[TMP0]], <4 x i1> splat (i1 true), <4 x double> poison)
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds double, ptr [[B_IN]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> align 8 [[TMP1]], <4 x i1> splat (i1 true), <4 x double> poison)
-; CHECK-NEXT: [[TMP2:%.*]] = call <4 x double> @llvm.sqrt.v4f64(<4 x double> [[WIDE_MASKED_GATHER1]])
-; CHECK-NEXT: br label %[[FOR2_HEADER2:.*]]
-; CHECK: [[FOR2_HEADER2]]:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[FOR2_HEADER2]] ]
-; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x double> [ [[WIDE_MASKED_GATHER]], %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[FOR2_HEADER2]] ]
-; CHECK-NEXT: [[TMP3]] = fadd <4 x double> [[TMP2]], [[VEC_PHI3]]
-; CHECK-NEXT: [[TMP4]] = add nuw nsw <4 x i32> [[VEC_PHI]], splat (i32 1)
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i32> [[TMP4]], splat (i32 10000)
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0
-; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[FOR2_HEADER2]]
-; CHECK: [[VECTOR_LATCH]]:
-; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds double, ptr [[C_OUT]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP3]], <4 x ptr> align 8 [[TMP7]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH:.*]]
-; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[SCALAR_PH:.*]]:
; CHECK-NEXT: br label %[[FOR1_HEADER:.*]]
; CHECK: [[FOR1_HEADER]]:
-; CHECK-NEXT: [[INDVAR1:%.*]] = phi i64 [ 1000, %[[SCALAR_PH]] ], [ [[INDVAR11:%.*]], %[[FOR1_LATCH:.*]] ]
+; CHECK-NEXT: [[INDVAR1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVAR11:%.*]], %[[FOR1_LATCH1:.*]] ]
; CHECK-NEXT: [[A_PTR:%.*]] = getelementptr inbounds double, ptr [[A_IN]], i64 [[INDVAR1]]
; CHECK-NEXT: [[A:%.*]] = load double, ptr [[A_PTR]], align 8
; CHECK-NEXT: [[B_PTR:%.*]] = getelementptr inbounds double, ptr [[B_IN]], i64 [[INDVAR1]]
@@ -49,19 +16,26 @@ define void @widen_call_instruction(ptr noalias nocapture readonly %a.in, ptr no
; CHECK-NEXT: [[B_SQRT:%.*]] = call double @llvm.sqrt.f64(double [[B]])
; CHECK-NEXT: br label %[[FOR2_HEADER:.*]]
; CHECK: [[FOR2_HEADER]]:
-; CHECK-NEXT: [[INDVAR2:%.*]] = phi i32 [ 0, %[[FOR1_HEADER]] ], [ [[INDVAR21:%.*]], %[[FOR2_HEADER]] ]
-; CHECK-NEXT: [[A_REDUCTION:%.*]] = phi double [ [[A]], %[[FOR1_HEADER]] ], [ [[A_REDUCTION1:%.*]], %[[FOR2_HEADER]] ]
-; CHECK-NEXT: [[A_REDUCTION1]] = fadd double [[B_SQRT]], [[A_REDUCTION]]
-; CHECK-NEXT: [[INDVAR21]] = add nuw nsw i32 [[INDVAR2]], 1
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x double> splat (double -0.000000e+00), double [[A]], i32 0
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[B_SQRT]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[FOR2_HEADER]] ], [ [[INDVAR21:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x double> [ [[TMP0]], %[[FOR2_HEADER]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP1]] = fadd <4 x double> [[BROADCAST_SPLAT]], [[VEC_PHI]]
+; CHECK-NEXT: [[INDVAR21]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[FOR2_COND:%.*]] = icmp eq i32 [[INDVAR21]], 10000
-; CHECK-NEXT: br i1 [[FOR2_COND]], label %[[FOR1_LATCH]], label %[[FOR2_HEADER]]
+; CHECK-NEXT: br i1 [[FOR2_COND]], label %[[FOR1_LATCH:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[FOR1_LATCH]]:
-; CHECK-NEXT: [[A_REDUCTION1_LCSSA:%.*]] = phi double [ [[A_REDUCTION1]], %[[FOR2_HEADER]] ]
+; CHECK-NEXT: [[A_REDUCTION1_LCSSA:%.*]] = call double @llvm.vector.reduce.fadd.v4f64(double -0.000000e+00, <4 x double> [[TMP1]])
+; CHECK-NEXT: br label %[[FOR1_LATCH1]]
+; CHECK: [[FOR1_LATCH1]]:
; CHECK-NEXT: [[C_PTR:%.*]] = getelementptr inbounds double, ptr [[C_OUT]], i64 [[INDVAR1]]
; CHECK-NEXT: store double [[A_REDUCTION1_LCSSA]], ptr [[C_PTR]], align 8
; CHECK-NEXT: [[INDVAR11]] = add nuw nsw i64 [[INDVAR1]], 1
; CHECK-NEXT: [[FOR1_COND:%.*]] = icmp eq i64 [[INDVAR11]], 1000
-; CHECK-NEXT: br i1 [[FOR1_COND]], label %[[EXIT]], label %[[FOR1_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT: br i1 [[FOR1_COND]], label %[[EXIT:.*]], label %[[FOR1_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -114,7 +88,7 @@ define void @call_to_non_intrinsic() {
; CHECK: [[OUTER_LATCH]]:
; CHECK-NEXT: [[OUTER_IV_NEXT]] = add i64 [[OUTER_IV]], 1
; CHECK-NEXT: [[OUTER_EC:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 100
-; CHECK-NEXT: br i1 [[OUTER_EC]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: br i1 [[OUTER_EC]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP3]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
@@ -149,7 +123,6 @@ declare void @use()
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]]}
-; CHECK: [[META5]] = !{!"llvm.loop.vectorize.enable", i1 true}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]]}
+; CHECK: [[META4]] = !{!"llvm.loop.vectorize.enable", i1 true}
;.
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll b/llvm/test/Transforms/LoopVectorize/widen-select.ll
similarity index 52%
rename from llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll
rename to llvm/test/Transforms/LoopVectorize/widen-select.ll
index 429e964890103..fc06db5c53b89 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll
+++ b/llvm/test/Transforms/LoopVectorize/widen-select.ll
@@ -1,47 +1,32 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph\:" --version 5
-; RUN: opt -passes=loop-vectorize -force-vector-width=4 -enable-vplan-native-path -S %s | FileCheck %s
-
-; Test that VPlan native path is able to widen select instruction in the
-; innermost loop under different conditions when outer loop is marked to be
-; vectorized. These conditions include following:
-; * Inner and outer loop invariant select condition
-; * Select condition depending on outer loop iteration variable.
-; * Select condidition depending on inner loop iteration variable.
-; * Select conditition depending on both outer and inner loop iteration
-; variables.
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
define void @loop_invariant_select(ptr noalias nocapture %out, i1 %select, double %a, double %b) {
; CHECK-LABEL: define void @loop_invariant_select(
; CHECK-SAME: ptr noalias captures(none) [[OUT:%.*]], i1 [[SELECT:%.*]], double [[A:%.*]], double [[B:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT2]], <4 x double> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[INDVAR1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVAR11:%.*]], %[[SCALAR_PH:.*]] ]
+; CHECK-NEXT: [[PTR:%.*]] = getelementptr inbounds double, ptr [[OUT]], i64 [[INDVAR1]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr [[OUT]], <4 x i64> [[VEC_IND]]
+; CHECK-NEXT: [[TMP0:%.*]] = select i1 [[SELECT]], double [[A]], double [[B]]
; CHECK-NEXT: br label %[[FOR2_HEADER3:.*]]
; CHECK: [[FOR2_HEADER3]]:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP2:%.*]], %[[FOR2_HEADER3]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[SELECT]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP1]], <4 x ptr> align 8 [[TMP0]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP2]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
-; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i64> [[TMP2]], splat (i64 10000)
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
-; CHECK-NEXT: br i1 [[TMP5]], label %[[VECTOR_LATCH]], label %[[FOR2_HEADER3]]
-; CHECK: [[VECTOR_LATCH]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[FOR2_HEADER3]] ]
+; CHECK-NEXT: store double [[TMP0]], ptr [[PTR]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
+; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR2_HEADER3]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH:.*]]
+; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[INDVAR11]] = add nuw nsw i64 [[INDVAR1]], 1
+; CHECK-NEXT: [[FOR1_COND:%.*]] = icmp eq i64 [[INDVAR11]], 1000
+; CHECK-NEXT: br i1 [[FOR1_COND]], label %[[EXIT:.*]], label %[[VECTOR_PH]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
;
entry:
br label %for1.header
@@ -72,36 +57,37 @@ exit:
define void @outer_loop_dependant_select(ptr noalias nocapture %out, double %a, double %b) {
; CHECK-LABEL: define void @outer_loop_dependant_select(
; CHECK-SAME: ptr noalias captures(none) [[OUT:%.*]], double [[A:%.*]], double [[B:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
-; CHECK: [[VECTOR_PH]]:
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT2]], <4 x double> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[VECTOR_PH:.*]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr [[OUT]], <4 x i64> [[VEC_IND]]
+; CHECK-NEXT: [[INDVAR1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDVAR11:%.*]], %[[SCALAR_PH:.*]] ]
+; CHECK-NEXT: [[PTR:%.*]] = getelementptr inbounds double, ptr [[OUT]], i64 [[INDVAR1]]
; CHECK-NEXT: br label %[[FOR2_HEADER3:.*]]
; CHECK: [[FOR2_HEADER3]]:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[FOR2_HEADER3]] ]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[B]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x double> poison, double [[A]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT1]], <4 x double> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i64> poison, i64 [[INDVAR1]], i64 0
+; CHECK-NEXT: [[VEC_IND:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT3]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP1:%.*]] = trunc <4 x i64> [[VEC_IND]] to <4 x i1>
; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP2]], <4 x ptr> align 8 [[TMP0]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP3]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP3]], splat (i64 10000)
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0
-; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[FOR2_HEADER3]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[TMP2]], i32 3
+; CHECK-NEXT: br label %[[VECTOR_LATCH:.*]]
; CHECK: [[VECTOR_LATCH]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[FOR2_HEADER3]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH]] ]
+; CHECK-NEXT: store double [[TMP4]], ptr [[PTR]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
+; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_LATCH]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH:.*]]
+; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[INDVAR11]] = add nuw nsw i64 [[INDVAR1]], 1
+; CHECK-NEXT: [[FOR1_COND:%.*]] = icmp eq i64 [[INDVAR11]], 1000
+; CHECK-NEXT: br i1 [[FOR1_COND]], label %[[EXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
;
entry:
br label %for1.header
@@ -133,36 +119,35 @@ exit:
define void @inner_loop_dependant_select(ptr noalias nocapture %out, double %a, double %b) {
; CHECK-LABEL: define void @inner_loop_dependant_select(
; CHECK-SAME: ptr noalias captures(none) [[OUT:%.*]], double [[A:%.*]], double [[B:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[INDVAR1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVAR11:%.*]], %[[SCALAR_PH:.*]] ]
+; CHECK-NEXT: [[PTR:%.*]] = getelementptr inbounds double, ptr [[OUT]], i64 [[INDVAR1]]
+; CHECK-NEXT: br label %[[VECTOR_PH1:.*]]
+; CHECK: [[VECTOR_PH1]]:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT2]], <4 x double> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr [[OUT]], <4 x i64> [[VEC_IND]]
; CHECK-NEXT: br label %[[FOR2_HEADER3:.*]]
; CHECK: [[FOR2_HEADER3]]:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[FOR2_HEADER3]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = trunc <4 x i64> [[VEC_PHI]] to <4 x i1>
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[FOR2_HEADER3]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <4 x i1> [ <i1 false, i1 true, i1 false, i1 true>, %[[VECTOR_PH1]] ], [ [[TMP1]], %[[FOR2_HEADER3]] ]
; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP2]], <4 x ptr> align 8 [[TMP0]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP3]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
-; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP3]], splat (i64 10000)
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0
-; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[FOR2_HEADER3]]
-; CHECK: [[VECTOR_LATCH]]:
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x double> [[TMP2]], i32 3
+; CHECK-NEXT: store double [[TMP3]], ptr [[PTR]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR2_HEADER3]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH:.*]]
+; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[INDVAR11]] = add nuw nsw i64 [[INDVAR1]], 1
+; CHECK-NEXT: [[FOR1_COND:%.*]] = icmp eq i64 [[INDVAR11]], 1000
+; CHECK-NEXT: br i1 [[FOR1_COND]], label %[[EXIT:.*]], label %[[VECTOR_PH]], !llvm.loop [[LOOP3]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
;
entry:
br label %for1.header
@@ -194,37 +179,40 @@ exit:
define void @outer_and_inner_loop_dependant_select(ptr noalias nocapture %out, double %a, double %b) {
; CHECK-LABEL: define void @outer_and_inner_loop_dependant_select(
; CHECK-SAME: ptr noalias captures(none) [[OUT:%.*]], double [[A:%.*]], double [[B:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[INDVAR1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDVAR11:%.*]], %[[SCALAR_PH:.*]] ]
+; CHECK-NEXT: [[PTR:%.*]] = getelementptr inbounds double, ptr [[OUT]], i64 [[INDVAR1]]
+; CHECK-NEXT: br label %[[VECTOR_PH1:.*]]
+; CHECK: [[VECTOR_PH1]]:
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[INDVAR1]], i64 0
+; CHECK-NEXT: [[VEC_IND:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT2]], <4 x double> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr [[OUT]], <4 x i64> [[VEC_IND]]
; CHECK-NEXT: br label %[[FOR2_HEADER3:.*]]
; CHECK: [[FOR2_HEADER3]]:
-; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[FOR2_HEADER3]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], %[[FOR2_HEADER3]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH1]] ], [ [[VEC_IND_NEXT:%.*]], %[[FOR2_HEADER3]] ]
; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], [[VEC_PHI]]
; CHECK-NEXT: [[TMP2:%.*]] = trunc <4 x i64> [[TMP1]] to <4 x i1>
; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]]
-; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP3]], <4 x ptr> align 8 [[TMP0]], <4 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP4]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
-; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP4]], splat (i64 10000)
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0
-; CHECK-NEXT: br i1 [[TMP7]], label %[[VECTOR_LATCH]], label %[[FOR2_HEADER3]]
-; CHECK: [[VECTOR_LATCH]]:
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[TMP3]], i32 3
+; CHECK-NEXT: store double [[TMP5]], ptr [[PTR]], align 8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 4)
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
+; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR2_HEADER3]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
-; CHECK-NEXT: br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH:.*]]
+; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
+; CHECK-NEXT: [[INDVAR11]] = add nuw nsw i64 [[INDVAR1]], 1
+; CHECK-NEXT: [[FOR1_COND:%.*]] = icmp eq i64 [[INDVAR11]], 1000
+; CHECK-NEXT: br i1 [[FOR1_COND]], label %[[EXIT:.*]], label %[[VECTOR_PH]], !llvm.loop [[LOOP3]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
;
entry:
br label %for1.header
More information about the llvm-commits
mailing list