[llvm] [VPlan] Split out adding canonical IV recipes to separate transform. (PR #197541)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Thu May 14 04:31:39 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/197541
>From 8c9af1d9d7adcd06932c8038d41ff3d44643cee3 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Tue, 12 May 2026 18:30:51 +0100
Subject: [PATCH] [VPlan] Split out adding canonical IV recipes to separate
transform.
Introduce canonical IV recipes after initial scalar
transformations/simplifications. Conceptually it is a separate
transformation and moving it later simplifies initial construction
The canonical IV is only needed once we handle early exits/introduce
regions.
This is needed to compute costs of scalar VPlans, where we need to
compare the cost of the original loop control instructions.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 9 ++---
.../Vectorize/VPlanConstruction.cpp | 34 +++++++------------
.../Transforms/Vectorize/VPlanTransforms.h | 9 +++--
.../VPlan/AArch64/call-decisions.ll | 4 +++
.../scalarize-wide-load-for-address-use.ll | 7 ++++
.../LoopVectorize/VPlan/find-last.ll | 2 ++
.../LoopVectorize/VPlan/predicator.ll | 14 ++++++++
.../LoopVectorize/VPlan/tail-folding.ll | 8 +++++
.../VPlan/vplan-print-after-all.ll | 1 +
.../Transforms/Vectorize/VPlanTestBase.h | 11 +++---
10 files changed, 67 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 7213d4ae795ec..d78716e2a1514 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6776,10 +6776,9 @@ void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF,
// Create initial base VPlan0, to serve as common starting point for all
// candidates built later for specific VF ranges.
- auto VPlan0 = VPlanTransforms::buildVPlan0(
- OrigLoop, *LI, Legal->getWidestInductionType(),
- getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE,
- LVer ? &*LVer : nullptr);
+ auto VPlan0 = VPlanTransforms::buildVPlan0(OrigLoop, *LI,
+ Legal->getWidestInductionType(),
+ PSE, LVer ? &*LVer : nullptr);
// Create recipes for header phis. For outer loops, reductions, recurrences
// and in-loop reductions are empty since legality doesn't detect them.
@@ -6792,6 +6791,8 @@ void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF,
RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, *VPlan0);
RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, *VPlan0);
+ RUN_VPLAN_PASS(VPlanTransforms::addCanonicalIVRecipes, *VPlan0,
+ getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()));
// If we're vectorizing a loop with an uncountable exit, make sure that the
// recipes are safe to handle.
// TODO: Remove this once we can properly check the VPlan itself for both
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 7c54a223f9793..bf1388678311d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -478,13 +478,14 @@ static void createLoopRegion(VPlan &Plan, VPBlockBase *HeaderVPB) {
VPBB->setParent(R);
}
-// Add the necessary canonical IV and branch recipes required to control the
-// loop.
-static void addCanonicalIVRecipes(VPlan &Plan, VPBasicBlock *HeaderVPBB,
- VPBasicBlock *LatchVPBB, Type *IdxTy,
- DebugLoc DL) {
+void VPlanTransforms::addCanonicalIVRecipes(VPlan &Plan, DebugLoc DL) {
+ auto *HeaderVPBB = cast<VPBasicBlock>(
+ Plan.getEntry()->getSuccessors()[1]->getSingleSuccessor());
+ auto *LatchVPBB = cast<VPBasicBlock>(HeaderVPBB->getPredecessors()[1]);
+
// Add a VPPhi for the canonical IV starting at 0 as first recipe in header.
- auto *CanonicalIVPHI = new VPPhi(Plan.getConstantInt(IdxTy, 0), {}, DL);
+ auto *CanonicalIVPHI =
+ new VPPhi(Plan.getZero(Plan.getVectorTripCount().getType()), {}, DL);
HeaderVPBB->insert(CanonicalIVPHI, HeaderVPBB->begin());
// We are about to replace the branch to exit the region. Remove the original
@@ -529,14 +530,7 @@ static void createExtractsForLiveOuts(VPlan &Plan, VPBasicBlock *MiddleVPBB) {
}
}
-/// Return an iterator range to iterate over pairs of matching phi nodes in
-/// \p Header and \p ScalarHeader, skipping the canonical IV in the former.
-static auto getMatchingPhisForScalarLoop(VPBasicBlock *Header,
- VPBasicBlock *ScalarHeader) {
- return zip_equal(drop_begin(Header->phis()), ScalarHeader->phis());
-}
-
-static void addInitialSkeleton(VPlan &Plan, Type *InductionTy, DebugLoc IVDL,
+static void addInitialSkeleton(VPlan &Plan, Type *InductionTy,
PredicatedScalarEvolution &PSE, Loop *TheLoop) {
VPDominatorTree VPDT(Plan);
@@ -560,8 +554,6 @@ static void addInitialSkeleton(VPlan &Plan, Type *InductionTy, DebugLoc IVDL,
LatchVPBB->swapSuccessors();
}
- addCanonicalIVRecipes(Plan, HeaderVPBB, LatchVPBB, InductionTy, IVDL);
-
// Create SCEV and VPValue for the trip count.
// We use the symbolic max backedge-taken-count, which works also when
// vectorizing loops with uncountable early exits.
@@ -596,7 +588,7 @@ static void addInitialSkeleton(VPlan &Plan, Type *InductionTy, DebugLoc IVDL,
ArrayRef<VPBlockBase *>({MiddleVPBB, Plan.getEntry()})) &&
"unexpected predecessor order of scalar ph");
for (const auto &[PhiR, ScalarPhiR] :
- getMatchingPhisForScalarLoop(HeaderVPBB, Plan.getScalarHeader())) {
+ zip_equal(HeaderVPBB->phis(), Plan.getScalarHeader()->phis())) {
auto *VectorPhiR = cast<VPPhi>(&PhiR);
VPValue *BackedgeVal = VectorPhiR->getOperand(1);
VPValue *ResumeFromVectorLoop =
@@ -634,11 +626,11 @@ static void printAfterInitialConstruction(VPlan &) {}
std::unique_ptr<VPlan>
VPlanTransforms::buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy,
- DebugLoc IVDL, PredicatedScalarEvolution &PSE,
+ PredicatedScalarEvolution &PSE,
LoopVersioning *LVer) {
PlainCFGBuilder Builder(TheLoop, &LI, LVer, InductionTy);
std::unique_ptr<VPlan> VPlan0 = Builder.buildPlainCFG();
- addInitialSkeleton(*VPlan0, InductionTy, IVDL, PSE, TheLoop);
+ addInitialSkeleton(*VPlan0, InductionTy, PSE, TheLoop);
simplifyLiveInsWithSCEV(*VPlan0, PSE);
RUN_VPLAN_PASS_NO_VERIFY(printAfterInitialConstruction, *VPlan0);
@@ -967,7 +959,7 @@ bool VPlanTransforms::createHeaderPhiRecipes(
RdxDesc.hasUsesOutsideReductionChain());
};
- for (VPRecipeBase &R : make_early_inc_range(drop_begin(HeaderVPBB->phis()))) {
+ for (VPRecipeBase &R : make_early_inc_range(HeaderVPBB->phis())) {
auto *PhiR = cast<VPPhi>(&R);
VPHeaderPHIRecipe *HeaderPhiR = CreateHeaderPhiRecipe(PhiR);
HeaderPhiR->insertBefore(PhiR);
@@ -979,7 +971,7 @@ bool VPlanTransforms::createHeaderPhiRecipes(
return false;
for (const auto &[HeaderPhiR, ScalarPhiR] :
- getMatchingPhisForScalarLoop(HeaderVPBB, Plan.getScalarPreheader())) {
+ zip_equal(HeaderVPBB->phis(), Plan.getScalarPreheader()->phis())) {
auto *ResumePhiR = cast<VPPhi>(&ScalarPhiR);
if (isa<VPFirstOrderRecurrencePHIRecipe>(&HeaderPhiR)) {
ResumePhiR->setName("scalar.recur.init");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index c66d83d3177d3..9aa9f17f864c2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -94,8 +94,7 @@ struct VPlanTransforms {
/// The created loop is wrapped in an initial skeleton to facilitate
/// vectorization, consisting of a vector pre-header, an exit block for the
/// main vector loop (middle.block) and a new block as preheader of the scalar
- /// loop (scalar.ph). See below for an illustration. It also adds a canonical
- /// IV and its increment, using \p InductionTy and \p IVDL, and creates a
+ /// loop (scalar.ph). See below for an illustration. It also creates a
/// VPValue expression for the original trip count.
///
/// [ ] <-- Plan's entry VPIRBasicBlock, wrapping the original loop's
@@ -129,9 +128,13 @@ struct VPlanTransforms {
/// \ v
/// >[ ] <-- original loop exit block(s), wrapped in VPIRBasicBlocks.
LLVM_ABI_FOR_TEST static std::unique_ptr<VPlan>
- buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy, DebugLoc IVDL,
+ buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy,
PredicatedScalarEvolution &PSE, LoopVersioning *LVer = nullptr);
+ /// Add a canonical IV and its increment, using \p InductionTy and \p DL to \p
+ /// Plan
+ static void addCanonicalIVRecipes(VPlan &Plan, DebugLoc DL);
+
/// Replace VPPhi recipes in \p Plan's header with corresponding
/// VPHeaderPHIRecipe subclasses for inductions, reductions, and
/// fixed-order recurrences. This processes all header phis and creates
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/call-decisions.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/call-decisions.ll
index 50c605d8aea62..81d4a1819ff04 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/call-decisions.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/call-decisions.ll
@@ -31,6 +31,8 @@ define void @pick_wide_call(ptr noalias %in.ptr, ptr noalias %out.ptr) {
; CHECK-NEXT: EMIT ir<%out.gep> = getelementptr inbounds ir<%out.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%out.gep>
; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<%call>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp eq ir<%iv.next>, ir<1000>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -83,6 +85,8 @@ define void @pick_intrinsic_call(ptr noalias %in.ptr, ptr noalias %out.ptr) {
; CHECK-NEXT: EMIT ir<%out.gep> = getelementptr inbounds ir<%out.ptr>, ir<%iv>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer inbounds ir<%out.gep>
; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<%call>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%exitcond> = icmp eq ir<%iv.next>, ir<1000>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/X86/scalarize-wide-load-for-address-use.ll b/llvm/test/Transforms/LoopVectorize/VPlan/X86/scalarize-wide-load-for-address-use.ll
index 02efbf0ac738b..4ef11fac79974 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/X86/scalarize-wide-load-for-address-use.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/X86/scalarize-wide-load-for-address-use.ll
@@ -33,6 +33,7 @@ define void @reverse_unmasked_load_feeds_address(ptr noalias %src, i64 %n) {
; CHECK-NEXT: EMIT ir<%cmp> = fcmp oeq ir<%val>, ir<0.000000e+00>
; CHECK-NEXT: EMIT ir<%ptr.sel> = select ir<%cmp>, ir<@tbl.a>, ir<@tbl.b>
; CHECK-NEXT: REPLICATE store ir<1.000000e+00>, ir<%ptr.sel>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv>, ir<0>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -90,6 +91,8 @@ define void @mixed_address_and_vector_uses(ptr noalias %src, ptr noalias %dst, i
; CHECK-NEXT: EMIT ir<%gep.dst> = getelementptr ir<%dst>, ir<%iv>
; CHECK-NEXT: vp<[[VP4:%[0-9]+]]> = vector-pointer ir<%gep.dst>
; CHECK-NEXT: WIDEN store vp<[[VP4]]>, ir<%doubled>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<%n>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -145,6 +148,8 @@ define void @interleave_member_feeds_address(ptr noalias %arr, i64 %n) {
; CHECK-NEXT: EMIT ir<%gep.v> = getelementptr ir<%arr>, ir<%iv>, ir<1>
; CHECK-NEXT: REPLICATE ir<%v> = load ir<%gep.v>
; CHECK-NEXT: REPLICATE store ir<%v>, ir<%p>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<%n>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
@@ -199,6 +204,8 @@ define void @symbolic_stride_versioned_to_one(ptr noalias %src, ptr noalias %dst
; CHECK-NEXT: EMIT ir<%gep.dst> = getelementptr ir<%dst>, ir<%iv>
; CHECK-NEXT: vp<[[VP5:%[0-9]+]]> = vector-pointer ir<%gep.dst>
; CHECK-NEXT: WIDEN store vp<[[VP5]]>, ir<%doubled>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<%n>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/find-last.ll b/llvm/test/Transforms/LoopVectorize/VPlan/find-last.ll
index d89aa48a27c7e..9360a4de91888 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/find-last.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/find-last.ll
@@ -29,6 +29,8 @@ define i32 @find_last_with_select(ptr noalias %a, ptr noalias %b) {
; CHECK-NEXT: WIDEN ir<%load.b> = load vp<[[VP5]]>
; CHECK-NEXT: WIDEN ir<%cmp> = icmp slt ir<%load.a>, ir<%load.b>
; CHECK-NEXT: WIDEN ir<%sel> = select ir<%cmp>, ir<%rdx>, ir<%load.a>
+; CHECK-NEXT: CLONE ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: CLONE ir<%exitcond> = icmp eq ir<%iv.next>, ir<500>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
index 8e05e27d185a6..2e955de703cdf 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/predicator.ll
@@ -24,6 +24,8 @@ define void @diamond_phi(ptr %a) {
; CHECK-NEXT: bb4:
; CHECK-NEXT: BLEND ir<%phi4> = ir<%add2>/vp<[[VP4]]> ir<%add1>/ir<%c0>
; CHECK-NEXT: EMIT store ir<%phi4>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK-NEXT: No successors
@@ -96,6 +98,8 @@ define void @mask_reuse(ptr %a) {
; CHECK-NEXT: EMIT vp<[[VP7:%[0-9]+]]> = not ir<%c0>
; CHECK-NEXT: BLEND ir<%phi4> = ir<%add3>/ir<%c0> ir<%iv>/vp<[[VP7]]>
; CHECK-NEXT: EMIT store ir<%phi4>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK-NEXT: No successors
@@ -202,6 +206,8 @@ define void @optimized_mask(ptr %a) {
; CHECK-NEXT: EMIT vp<[[VP16:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP15]]>
; CHECK-NEXT: BLEND ir<%phi7> = ir<%add6>/vp<[[VP16]]> ir<%add5>/vp<[[VP14]]>
; CHECK-NEXT: EMIT store ir<%phi7>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK-NEXT: No successors
@@ -316,6 +322,8 @@ define void @switch(ptr %a) {
; CHECK-NEXT: EMIT vp<[[VP17:%[0-9]+]]> = logical-and vp<[[VP4]]>, vp<[[VP16]]>
; CHECK-NEXT: BLEND ir<%phi5> = ir<%add4>/vp<[[VP9]]> ir<%add3>/vp<[[VP15]]> ir<%add2>/vp<[[VP17]]> ir<%add1>/vp<[[VP14]]>
; CHECK-NEXT: EMIT store ir<%phi5>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK-NEXT: No successors
@@ -414,6 +422,8 @@ define void @diamond_phi2(ptr %a, i1 %c1, i1 %c2) {
; CHECK-NEXT: Successor(s): bb5
; CHECK-EMPTY:
; CHECK-NEXT: bb5:
+; CHECK-NEXT: EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK-NEXT: No successors
@@ -513,6 +523,8 @@ define void @blend_masks(ptr noalias %p, i1 %c0, i1 %c1, i1 %c2, i1 %c3, i1 %c4)
; CHECK-NEXT: Successor(s): bb8
; CHECK-EMPTY:
; CHECK-NEXT: bb8:
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK-NEXT: No successors
@@ -594,6 +606,8 @@ define void @blend_masks_triangle_phi(ptr noalias %p, i1 %c0, i1 %c1) {
; CHECK-NEXT: BLEND ir<%phi> = ir<1>/vp<[[VP7]]> ir<0>/vp<[[VP8]]>
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
; CHECK-NEXT: EMIT store ir<%phi>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<128>
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP3]]>, vp<[[VP1:%[0-9]+]]>
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2:%[0-9]+]]>
; CHECK-NEXT: No successors
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll b/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
index 04b7c612f4587..f30cb227bd967 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
@@ -31,6 +31,8 @@ define i32 @live_out(ptr noalias %p, i32 %n) {
; CHECK-NEXT: EMIT-SCALAR ir<%x> = load ir<%gep>
; CHECK-NEXT: EMIT ir<%y> = add ir<%x>, ir<1>
; CHECK-NEXT: EMIT store ir<%y>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<%n>
; CHECK-NEXT: Successor(s): vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.latch:
@@ -124,6 +126,8 @@ define i32 @conditional_live_out(ptr noalias %p, i32 %n, i1 %c) {
; CHECK-EMPTY:
; CHECK-NEXT: latch:
; CHECK-NEXT: EMIT-SCALAR ir<%phi> = phi [ ir<%y>, if ], [ ir<0>, vector.body.split ]
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<%n>
; CHECK-NEXT: Successor(s): vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.latch:
@@ -209,6 +213,8 @@ define void @header_unconditional_branch(ptr noalias %p, i32 %n) {
; CHECK-NEXT: Successor(s): latch
; CHECK-EMPTY:
; CHECK-NEXT: latch:
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<%n>
; CHECK-NEXT: Successor(s): vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.latch:
@@ -281,6 +287,8 @@ define i32 @reduction(ptr noalias %p, i32 %n) {
; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
; CHECK-NEXT: EMIT-SCALAR ir<%x> = load ir<%gep>
; CHECK-NEXT: EMIT ir<%rdx.next> = add ir<%rdx>, ir<%x>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<%n>
; CHECK-NEXT: Successor(s): vector.latch
; CHECK-EMPTY:
; CHECK-NEXT: vector.latch:
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
index fcaf5f4867ef7..3e863b574cf43 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
@@ -7,6 +7,7 @@
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createHeaderPhiRecipes
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::simplifyRecipes
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::removeDeadRecipes
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::addCanonicalIVRecipes
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::handleEarlyExits
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::addMiddleCheck
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createLoopRegions
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
index 4f8f3581afa8c..73776c1f901ca 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
@@ -78,8 +78,8 @@ class VPlanTestIRBase : public testing::Test {
Loop *L = LI->getLoopFor(LoopHeader);
PredicatedScalarEvolution PSE(*SE, *L);
- auto Plan = VPlanTransforms::buildVPlan0(L, *LI, IntegerType::get(*Ctx, 64),
- {}, PSE);
+ auto Plan =
+ VPlanTransforms::buildVPlan0(L, *LI, IntegerType::get(*Ctx, 64), PSE);
if (Style != UncountableExitStyle::NoUncountableExit) {
Inductions.clear();
@@ -96,6 +96,7 @@ class VPlanTestIRBase : public testing::Test {
/*AllowReordering=*/false);
}
+ VPlanTransforms::addCanonicalIVRecipes(*Plan, {});
VPlanTransforms::handleEarlyExits(*Plan, Style, L, PSE, *DT, AC.get());
VPlanTransforms::addMiddleCheck(*Plan, false);
@@ -111,8 +112,10 @@ class VPlanTestIRBase : public testing::Test {
Loop *L = LI->getLoopFor(LoopHeader);
PredicatedScalarEvolution PSE(*SE, *L);
- return VPlanTransforms::buildVPlan0(L, *LI, IntegerType::get(*Ctx, 64), {},
- PSE);
+ auto Plan =
+ VPlanTransforms::buildVPlan0(L, *LI, IntegerType::get(*Ctx, 64), PSE);
+ VPlanTransforms::addCanonicalIVRecipes(*Plan, {});
+ return Plan;
}
};
More information about the llvm-commits
mailing list