[llvm] 8251293 - [VPlan] Move tail folding out of VPlanPredicator. NFC (#176143)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Mar 5 00:17:47 PST 2026
Author: Luke Lau
Date: 2026-03-05T08:17:37Z
New Revision: 825129378e0f1a6671de77aa8e221a4838fd8219
URL: https://github.com/llvm/llvm-project/commit/825129378e0f1a6671de77aa8e221a4838fd8219
DIFF: https://github.com/llvm/llvm-project/commit/825129378e0f1a6671de77aa8e221a4838fd8219.diff
LOG: [VPlan] Move tail folding out of VPlanPredicator. NFC (#176143)
Currently the logic for introducing a header mask and predicating the
vector loop region is done inside introduceMasksAndLinearize.
This splits the tail folding part out into an individual VPlan transform
so that VPlanPredicator.cpp doesn't need to worry about tail folding,
which seemed to be a temporary measure according to a comment in
VPlanTransforms.h.
To perform tail folding independently, this splits the "body" of the
vector loop region between the phis in the header and the branch + iv
increment in the latch:
Before:
```
+-------------------------------------------+
|%iv = ... |
|... |
|%iv.next = add %iv, vfxuf |
|branch-on-count %iv.next, vector-trip-count|
+-------------------------------------------+
```
After:
```
+-------------------------------------------+
|%iv = ... |
|%wide.iv = widen-canonical-iv ... |
|%header-mask = icmp ule %wide.iv, BTC |---+
|branch-on-cond %header-mask | |
+-------------------------------------------+ |
| |
v |
+-------------------------------------------+ |
|... | |
+-------------------------------------------+ |
| |
v |
+-------------------------------------------+ |
|%iv.next = add %iv, vfxuf |<--+
|branch-on-count %iv.next, vector-trip-count|
+-------------------------------------------+
```
Phis are then inserted in the latch for any value in the loop body that
have outside uses, with poison as their incoming value from the header
edge.
The motivation for this is to allow us to share the same "predicate all
successor blocks" type of predication we do for tail folding, but for
early-exit loops in #172454. This may also allow us to directly emit an
EVL based header mask, instead of having to match + transform the
existing header mask in addExplicitVectorLength.
This also allows us to eventually handle recurrences in the same
transform, avoiding the need to special case tail folding in
addReductionResultComputation.
Added:
llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.cpp
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.h
llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 91c7f1680aac2..6104e1a6aa1d4 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8183,7 +8183,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
VPlanTransforms::addMiddleCheck(*Plan, RequiresScalarEpilogueCheck,
CM.foldTailByMasking());
- VPlanTransforms::createLoopRegions(*Plan);
+ RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::createLoopRegions, *Plan);
+ if (CM.foldTailByMasking())
+ RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::foldTailByMasking, *Plan);
// Don't use getDecisionAndClampRange here, because we don't know the UF
// so this function is better to be conservative, rather than to split
@@ -8238,8 +8240,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// ---------------------------------------------------------------------------
// Predicate and linearize the top-level loop region.
// ---------------------------------------------------------------------------
- RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::introduceMasksAndLinearize, *Plan,
- CM.foldTailByMasking());
+ RUN_VPLAN_PASS_NO_VERIFY(VPlanTransforms::introduceMasksAndLinearize, *Plan);
// ---------------------------------------------------------------------------
// Construct wide recipes and apply predication for original scalar
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 749702af0265b..0ceeb570e8b1f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -570,6 +570,11 @@ VPBasicBlock *VPBasicBlock::splitAt(iterator SplitAt) {
auto *SplitBlock = getPlan()->createVPBasicBlock(getName() + ".split");
VPBlockUtils::insertBlockAfter(SplitBlock, this);
+ // If this is the exiting block, make the split the new exiting block.
+ auto *ParentRegion = getParent();
+ if (ParentRegion && ParentRegion->getExiting() == this)
+ ParentRegion->setExiting(SplitBlock);
+
// Finally, move the recipes starting at SplitAt to new block.
for (VPRecipeBase &ToMove :
make_early_inc_range(make_range(SplitAt, this->end())))
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 83907fb96dbd2..a4a0e5d9a8b4d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -991,6 +991,96 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan) {
TopRegion->getEntryBasicBlock()->setName("vector.body");
}
+void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
+ assert(Plan.getExitBlocks().size() == 1 &&
+ "only a single-exit block is supported currently");
+ assert(Plan.getExitBlocks().front()->getSinglePredecessor() ==
+ Plan.getMiddleBlock() &&
+ "the exit block must have middle block as single predecessor");
+
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ assert(LoopRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
+ "The vector loop region must have the middle block as its single "
+ "successor for now");
+ VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
+
+ Header->splitAt(Header->getFirstNonPhi());
+
+ // Create the header mask, insert it in the header and branch on it.
+ auto *IV =
+ new VPWidenCanonicalIVRecipe(Header->getParent()->getCanonicalIV());
+ VPBuilder Builder(Header, Header->getFirstNonPhi());
+ Builder.insert(IV);
+ VPValue *BTC = Plan.getOrCreateBackedgeTakenCount();
+ VPValue *HeaderMask = Builder.createICmp(CmpInst::ICMP_ULE, IV, BTC);
+ Builder.createNaryOp(VPInstruction::BranchOnCond, HeaderMask);
+
+ VPBasicBlock *OrigLatch = LoopRegion->getExitingBasicBlock();
+ VPValue *IVInc;
+ [[maybe_unused]] bool TermBranchOnCount =
+ match(OrigLatch->getTerminator(),
+ m_BranchOnCount(m_VPValue(IVInc),
+ m_Specific(&Plan.getVectorTripCount())));
+ assert(TermBranchOnCount &&
+ match(IVInc, m_Add(m_Specific(LoopRegion->getCanonicalIV()),
+ m_Specific(&Plan.getVFxUF()))) &&
+ std::next(IVInc->getDefiningRecipe()->getIterator()) ==
+ OrigLatch->getTerminator()->getIterator() &&
+ "Unexpected canonical iv increment");
+
+ // Split the latch at the IV update, and branch to it from the header mask.
+ VPBasicBlock *Latch =
+ OrigLatch->splitAt(IVInc->getDefiningRecipe()->getIterator());
+ Latch->setName("vector.latch");
+ VPBlockUtils::connectBlocks(Header, Latch);
+
+ // Collect any values defined in the loop that need a phi. Currently this
+ // includes header phi backedges and live-outs extracted in the middle block.
+ // TODO: Handle early exits via Plan.getExitBlocks()
+ MapVector<VPValue *, SmallVector<VPUser *>> NeedsPhi;
+ for (VPRecipeBase &R : Header->phis())
+ if (!isa<VPCanonicalIVPHIRecipe, VPWidenInductionRecipe>(R))
+ NeedsPhi[cast<VPHeaderPHIRecipe>(R).getBackedgeValue()].push_back(&R);
+
+ VPValue *V;
+ for (VPRecipeBase &R : *Plan.getMiddleBlock())
+ if (match(&R, m_ExtractLastPart(m_VPValue(V))))
+ NeedsPhi[V].push_back(&R);
+
+ // Insert phis with a poison incoming value for past the end of the tail.
+ Builder.setInsertPoint(Latch, Latch->begin());
+ VPTypeAnalysis TypeInfo(Plan);
+ for (const auto &[V, Users] : NeedsPhi) {
+ if (isa<VPIRValue>(V))
+ continue;
+ // TODO: For reduction phis, use phi value instead of poison so we can
+ // remove the special casing for tail folding in
+ // LoopVectorizationPlanner::addReductionResultComputation
+ VPValue *Poison =
+ Plan.getOrAddLiveIn(PoisonValue::get(TypeInfo.inferScalarType(V)));
+ VPInstruction *Phi = Builder.createScalarPhi({V, Poison});
+ for (VPUser *U : Users)
+ U->replaceUsesOfWith(V, Phi);
+ }
+
+ // Any extract of the last element must be updated to extract from the last
+ // active lane of the header mask instead (i.e., the lane corresponding to the
+ // last active iteration).
+ Builder.setInsertPoint(Plan.getMiddleBlock()->getTerminator());
+ for (VPRecipeBase &R : *Plan.getMiddleBlock()) {
+ VPValue *Op;
+ if (!match(&R, m_ExtractLastLaneOfLastPart(m_VPValue(Op))))
+ continue;
+
+ // Compute the index of the last active lane.
+ VPValue *LastActiveLane =
+ Builder.createNaryOp(VPInstruction::LastActiveLane, HeaderMask);
+ auto *Ext =
+ Builder.createNaryOp(VPInstruction::ExtractLane, {LastActiveLane, Op});
+ R.getVPSingleValue()->replaceAllUsesWith(Ext);
+ }
+}
+
/// Insert \p CheckBlockVPBB on the edge leading to the vector preheader,
/// connecting it to both vector and scalar preheaders. Updates scalar
/// preheader phis to account for the new predecessor.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 1ab29d5140661..f22a33fa8eec3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -73,11 +73,7 @@ class VPPredicator {
return EdgeMaskCache.lookup({Src, Dst});
}
- /// Compute and return the mask for the vector loop header block.
- void createHeaderMask(VPBasicBlock *HeaderVPBB, bool FoldTail);
-
- /// Compute the predicate of \p VPBB, assuming that the header block of the
- /// loop is set to True, or to the loop mask when tail folding.
+ /// Compute the predicate of \p VPBB.
void createBlockInMask(VPBasicBlock *VPBB);
/// Convert phi recipes in \p VPBB to VPBlendRecipes.
@@ -154,28 +150,6 @@ void VPPredicator::createBlockInMask(VPBasicBlock *VPBB) {
setBlockInMask(VPBB, BlockMask);
}
-void VPPredicator::createHeaderMask(VPBasicBlock *HeaderVPBB, bool FoldTail) {
- if (!FoldTail) {
- setBlockInMask(HeaderVPBB, nullptr);
- return;
- }
-
- // Introduce the early-exit compare IV <= BTC to form header block mask.
- // This is used instead of IV < TC because TC may wrap, unlike BTC. Start by
- // constructing the desired canonical IV in the header block as its first
- // non-phi instructions.
-
- auto &Plan = *HeaderVPBB->getPlan();
- auto *IV =
- new VPWidenCanonicalIVRecipe(HeaderVPBB->getParent()->getCanonicalIV());
- Builder.setInsertPoint(HeaderVPBB, HeaderVPBB->getFirstNonPhi());
- Builder.insert(IV);
-
- VPValue *BTC = Plan.getOrCreateBackedgeTakenCount();
- VPValue *BlockMask = Builder.createICmp(CmpInst::ICMP_ULE, IV, BTC);
- setBlockInMask(HeaderVPBB, BlockMask);
-}
-
void VPPredicator::createSwitchEdgeMasks(const VPInstruction *SI) {
const VPBasicBlock *Src = SI->getParent();
@@ -266,7 +240,7 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
}
}
-void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan, bool FoldTail) {
+void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
// Scan the body of the loop in a topological order to visit each basic block
// after having visited its predecessor basic blocks.
@@ -280,9 +254,7 @@ void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan, bool FoldTail) {
// Introduce the mask for VPBB, which may introduce needed edge masks, and
// convert all phi recipes of VPBB to blend recipes unless VPBB is the
// header.
- if (VPBB == Header) {
- Predicator.createHeaderMask(Header, FoldTail);
- } else {
+ if (VPBB != Header) {
Predicator.createBlockInMask(VPBB);
Predicator.convertPhisToBlends(VPBB);
}
@@ -314,31 +286,4 @@ void VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan, bool FoldTail) {
PrevVPBB = VPBB;
}
-
- // If we folded the tail and introduced a header mask, any extract of the
- // last element must be updated to extract from the last active lane of the
- // header mask instead (i.e., the lane corresponding to the last active
- // iteration).
- if (FoldTail) {
- assert(Plan.getExitBlocks().size() == 1 &&
- "only a single-exit block is supported currently");
- assert(Plan.getExitBlocks().front()->getSinglePredecessor() ==
- Plan.getMiddleBlock() &&
- "the exit block must have middle block as single predecessor");
-
- VPBuilder B(Plan.getMiddleBlock()->getTerminator());
- for (VPRecipeBase &R : *Plan.getMiddleBlock()) {
- VPValue *Op;
- if (!match(&R, m_ExtractLastLane(m_ExtractLastPart(m_VPValue(Op)))))
- continue;
-
- // Compute the index of the last active lane.
- VPValue *HeaderMask = Predicator.getBlockInMask(Header);
- VPValue *LastActiveLane =
- B.createNaryOp(VPInstruction::LastActiveLane, HeaderMask);
- auto *Ext =
- B.createNaryOp(VPInstruction::ExtractLane, {LastActiveLane, Op});
- R.getVPSingleValue()->replaceAllUsesWith(Ext);
- }
- }
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 16f7ae2daeb5e..008f84f4be363 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -444,10 +444,44 @@ struct VPlanTransforms {
static std::unique_ptr<VPlan>
narrowInterleaveGroups(VPlan &Plan, const TargetTransformInfo &TTI);
+ /// Adapts the vector loop region for tail folding by introducing a header
+ /// mask and conditionally executing the content of the region:
+ ///
+ /// Vector loop region before:
+ /// +-------------------------------------------+
+ /// |%iv = ... |
+ /// |... |
+ /// |%iv.next = add %iv, vfxuf |
+ /// |branch-on-count %iv.next, vector-trip-count|
+ /// +-------------------------------------------+
+ ///
+ /// Vector loop region after:
+ /// +-------------------------------------------+
+ /// |%iv = ... |
+ /// |%wide.iv = widen-canonical-iv ... |
+ /// |%header-mask = icmp ule %wide.iv, BTC |
+ /// |branch-on-cond %header-mask |---+
+ /// +-------------------------------------------+ |
+ /// | |
+ /// v |
+ /// +-------------------------------------------+ |
+ /// | ... | |
+ /// +-------------------------------------------+ |
+ /// | |
+ /// v |
+ /// +-------------------------------------------+ |
+ /// |<phis> = phi [..., ...], [poison, header] |
+ /// |%iv.next = add %iv, vfxuf |<--+
+ /// |branch-on-count %iv.next, vector-trip-count|
+ /// +-------------------------------------------+
+ ///
+ /// Any VPInstruction::ExtractLastLanes are also updated to extract from the
+ /// last active lane of the header mask.
+ static void foldTailByMasking(VPlan &Plan);
+
/// Predicate and linearize the control-flow in the only loop region of
- /// \p Plan. If \p FoldTail is true, create a mask guarding the loop
- /// header, otherwise use all-true for the header mask.
- static void introduceMasksAndLinearize(VPlan &Plan, bool FoldTail);
+ /// \p Plan.
+ static void introduceMasksAndLinearize(VPlan &Plan);
/// Add branch weight metadata, if the \p Plan's middle block is terminated by
/// a BranchOnCond recipe.
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll b/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
new file mode 100644
index 0000000000000..6f80a678f3a50
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/tail-folding.ll
@@ -0,0 +1,338 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -p loop-vectorize -force-vector-width=4 -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S -vplan-print-after=foldTailByMasking -disable-output 2>&1 | FileCheck %s
+
+define i32 @live_out(ptr noalias %p, i32 %n) {
+; CHECK-LABEL: VPlan for loop in 'live_out'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in vp<[[VP3:%[0-9]+]]> = backedge-taken count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<[[VP4]]>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = icmp ule vp<[[VP5]]>, vp<[[VP3]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): vector.body.split, vector.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body.split:
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%x> = load ir<%gep>
+; CHECK-NEXT: EMIT ir<%y> = add ir<%x>, ir<1>
+; CHECK-NEXT: EMIT store ir<%y>, ir<%gep>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<%n>
+; CHECK-NEXT: Successor(s): vector.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.latch:
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP8:%[0-9]+]]> = phi [ ir<%y>, vector.body.split ], [ ir<poison>, vector.body ]
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = extract-last-part vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-lane vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = last-active-lane vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-lane vp<[[VP12]]>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT branch-on-cond ir<true>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %y.lcssa = phi i32 [ %y, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP15:%[0-9]+]]> = phi [ ir<%iv>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<[[VP15]]> from scalar.ph)
+; CHECK-NEXT: IR %gep = getelementptr i32, ptr %p, i32 %iv
+; CHECK-NEXT: IR %x = load i32, ptr %gep, align 4
+; CHECK-NEXT: IR %y = add i32 %x, 1
+; CHECK-NEXT: IR store i32 %y, ptr %gep, align 4
+; CHECK-NEXT: IR %iv.next = add i32 %iv, 1
+; CHECK-NEXT: IR %ec = icmp eq i32 %iv.next, %n
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ %x = load i32, ptr %gep
+ %y = add i32 %x, 1
+ store i32 %y, ptr %gep
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret i32 %y
+}
+
+define i32 @conditional_live_out(ptr noalias %p, i32 %n, i1 %c) {
+; CHECK-LABEL: VPlan for loop in 'conditional_live_out'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in vp<[[VP3:%[0-9]+]]> = backedge-taken count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<[[VP4]]>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = icmp ule vp<[[VP5]]>, vp<[[VP3]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): vector.body.split, vector.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body.split:
+; CHECK-NEXT: EMIT branch-on-cond ir<%c>
+; CHECK-NEXT: Successor(s): if, latch
+; CHECK-EMPTY:
+; CHECK-NEXT: if:
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%x> = load ir<%gep>
+; CHECK-NEXT: EMIT ir<%y> = add ir<%x>, ir<1>
+; CHECK-NEXT: EMIT store ir<%y>, ir<%gep>
+; CHECK-NEXT: Successor(s): latch
+; CHECK-EMPTY:
+; CHECK-NEXT: latch:
+; CHECK-NEXT: EMIT-SCALAR ir<%phi> = phi [ ir<%y>, if ], [ ir<0>, vector.body.split ]
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<%n>
+; CHECK-NEXT: Successor(s): vector.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.latch:
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP8:%[0-9]+]]> = phi [ ir<%phi>, latch ], [ ir<poison>, vector.body ]
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = extract-last-part vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-lane vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = last-active-lane vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-lane vp<[[VP12]]>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT branch-on-cond ir<true>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %phi.lcssa = phi i32 [ %phi, %latch ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP15:%[0-9]+]]> = phi [ ir<%iv>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: vp<[[VP15]]> from scalar.ph)
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %latch]
+ br i1 %c, label %if, label %latch
+
+if:
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ %x = load i32, ptr %gep
+ %y = add i32 %x, 1
+ store i32 %y, ptr %gep
+ br label %latch
+
+latch:
+ %phi = phi i32 [0, %loop], [%y, %if]
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret i32 %phi
+}
+
+define void @header_unconditional_branch(ptr noalias %p, i32 %n) {
+; CHECK-LABEL: VPlan for loop in 'header_unconditional_branch'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in vp<[[VP3:%[0-9]+]]> = backedge-taken count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<[[VP4]]>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = icmp ule vp<[[VP5]]>, vp<[[VP3]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): vector.body.split, vector.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body.split:
+; CHECK-NEXT: Successor(s): latch
+; CHECK-EMPTY:
+; CHECK-NEXT: latch:
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<%n>
+; CHECK-NEXT: Successor(s): vector.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.latch:
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT branch-on-cond ir<true>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP10:%[0-9]+]]> = phi [ ir<%iv>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ] (extra operand: vp<[[VP10]]> from scalar.ph)
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %latch]
+ br label %latch
+
+latch:
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+define i32 @reduction(ptr noalias %p, i32 %n) {
+; CHECK-LABEL: VPlan for loop in 'reduction'
+; CHECK: VPlan ' for UF>=1' {
+; CHECK-NEXT: Live-in vp<[[VP0:%[0-9]+]]> = VF
+; CHECK-NEXT: Live-in vp<[[VP1:%[0-9]+]]> = VF * UF
+; CHECK-NEXT: Live-in vp<[[VP2:%[0-9]+]]> = vector-trip-count
+; CHECK-NEXT: Live-in vp<[[VP3:%[0-9]+]]> = backedge-taken count
+; CHECK-NEXT: Live-in ir<%n> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<entry>:
+; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT: vector.body:
+; CHECK-NEXT: EMIT vp<[[VP4:%[0-9]+]]> = CANONICAL-INDUCTION ir<0>, vp<%index.next>
+; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VP0]]>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%rdx> = phi ir<0>, vp<[[VP8:%[0-9]+]]>
+; CHECK-NEXT: EMIT vp<[[VP5:%[0-9]+]]> = WIDEN-CANONICAL-INDUCTION vp<[[VP4]]>
+; CHECK-NEXT: EMIT vp<[[VP6:%[0-9]+]]> = icmp ule vp<[[VP5]]>, vp<[[VP3]]>
+; CHECK-NEXT: EMIT branch-on-cond vp<[[VP6]]>
+; CHECK-NEXT: Successor(s): vector.body.split, vector.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.body.split:
+; CHECK-NEXT: EMIT ir<%gep> = getelementptr ir<%p>, ir<%iv>
+; CHECK-NEXT: EMIT-SCALAR ir<%x> = load ir<%gep>
+; CHECK-NEXT: EMIT ir<%rdx.next> = add ir<%rdx>, ir<%x>
+; CHECK-NEXT: EMIT ir<%iv.next> = add ir<%iv>, ir<1>
+; CHECK-NEXT: EMIT ir<%ec> = icmp eq ir<%iv.next>, ir<%n>
+; CHECK-NEXT: Successor(s): vector.latch
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.latch:
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP8]]> = phi [ ir<%rdx.next>, vector.body.split ], [ ir<poison>, vector.body ]
+; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[VP4]]>, vp<[[VP1]]>
+; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<[[VP2]]>
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+; CHECK-NEXT: Successor(s): middle.block
+; CHECK-EMPTY:
+; CHECK-NEXT: middle.block:
+; CHECK-NEXT: EMIT vp<[[VP10:%[0-9]+]]> = extract-last-part vp<[[VP8]]>
+; CHECK-NEXT: EMIT vp<[[VP11:%[0-9]+]]> = extract-last-lane vp<[[VP10]]>
+; CHECK-NEXT: EMIT vp<[[VP12:%[0-9]+]]> = last-active-lane vp<[[VP6]]>
+; CHECK-NEXT: EMIT vp<[[VP13:%[0-9]+]]> = extract-lane vp<[[VP12]]>, vp<[[VP8]]>
+; CHECK-NEXT: EMIT branch-on-cond ir<true>
+; CHECK-NEXT: Successor(s): ir-bb<exit>, scalar.ph
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<exit>:
+; CHECK-NEXT: IR %rdx.next.lcssa = phi i32 [ %rdx.next, %loop ] (extra operand: vp<[[VP13]]> from middle.block)
+; CHECK-NEXT: No successors
+; CHECK-EMPTY:
+; CHECK-NEXT: scalar.ph:
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP15:%[0-9]+]]> = phi [ ir<%iv>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: EMIT-SCALAR vp<[[VP16:%[0-9]+]]> = phi [ ir<%rdx>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT: Successor(s): ir-bb<loop>
+; CHECK-EMPTY:
+; CHECK-NEXT: ir-bb<loop>:
+; CHECK-NEXT: IR %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] (extra operand: vp<[[VP15]]> from scalar.ph)
+; CHECK-NEXT: IR %rdx = phi i32 [ 0, %entry ], [ %rdx.next, %loop ] (extra operand: vp<[[VP16]]> from scalar.ph)
+; CHECK-NEXT: IR %gep = getelementptr i32, ptr %p, i32 %iv
+; CHECK-NEXT: IR %x = load i32, ptr %gep, align 4
+; CHECK-NEXT: IR %rdx.next = add i32 %rdx, %x
+; CHECK-NEXT: IR %iv.next = add i32 %iv, 1
+; CHECK-NEXT: IR %ec = icmp eq i32 %iv.next, %n
+; CHECK-NEXT: No successors
+; CHECK-NEXT: }
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i32 [0, %entry], [%iv.next, %loop]
+ %rdx = phi i32 [0, %entry], [%rdx.next, %loop]
+ %gep = getelementptr i32, ptr %p, i32 %iv
+ %x = load i32, ptr %gep
+ %rdx.next = add i32 %rdx, %x
+ %iv.next = add i32 %iv, 1
+ %ec = icmp eq i32 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret i32 %rdx.next
+}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
index bc9367942ac27..cba0948478955 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
@@ -4,6 +4,7 @@
; Verify that `-vplan-print-after-all` option works.
; CHECK: VPlan for loop in 'foo' after printAfterInitialConstruction
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createLoopRegions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::introduceMasksAndLinearize
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::clearReductionWrapFlags
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::optimizeFindIVReductions
More information about the llvm-commits
mailing list