[llvm] [VPlan] Move tail folding out of VPlanPredicator. NFC (PR #176143)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Tue Feb 3 02:14:13 PST 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/176143
>From 8e7238ebcd31161d2272a669dd394dfb6c42c3d5 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 15 Jan 2026 20:10:57 +0800
Subject: [PATCH 1/9] [VPlan] Move tail folding out of VPlanPredicator. NFC
Currently the logic for introducing a header mask and predicating the vector loop region is done inside introduceMasksAndLinearize.
This splits the tail folding part out into an individual VPlan transform so that VPlanPredicator.cpp doesn't need to worry about tail folding, which seemed to be a temporary measure according to a comment in VPlanTransforms.h.
To perform tail folding independently, this splits the "body" of the vector loop region between the phis in the header and the branch + iv increment in the latch:
Before:
+-------------------------------------------+
|%iv = ... |
|... |
|%iv.next = add %iv, vfxuf |
|branch-on-count %iv.next, vector-trip-count|
+-------------------------------------------+
After:
+-------------------------------------------+
|%iv = ... |
|%wide.iv = widen-canonical-iv ... |
|%header-mask = icmp ult %wide.iv, BTC |---+
|branch-on-cond %header-mask | |
+-------------------------------------------+ |
| |
v |
+-------------------------------------------+ |
|... | |
+-------------------------------------------+ |
| |
v |
+-------------------------------------------+ |
|%iv.next = add %iv, vfxuf |<--+
|branch-on-count %iv.next, vector-trip-count|
+-------------------------------------------+
The motivation for this is to align tail folding predication with early-exit predication in #172454. This will also allow us to directly emit an EVL based header mask, instead of having to match + transform the existing header mask in addExplicitVectorLength.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 9 ++-
llvm/lib/Transforms/Vectorize/VPlan.cpp | 5 ++
.../Vectorize/VPlanConstruction.cpp | 55 +++++++++++++++
.../Transforms/Vectorize/VPlanPredicator.cpp | 67 +------------------
.../Transforms/Vectorize/VPlanTransforms.h | 42 ++++++++++--
5 files changed, 105 insertions(+), 73 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 3f1e12e5d1cd0..1dcd33828445a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8463,11 +8463,13 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
InterleaveGroups.insert(IG);
}
+ if (CM.foldTailByMasking())
+ VPlanTransforms::foldTailByMasking(*Plan);
+
// ---------------------------------------------------------------------------
// Predicate and linearize the top-level loop region.
// ---------------------------------------------------------------------------
- auto BlockMaskCache = VPlanTransforms::introduceMasksAndLinearize(
- *Plan, CM.foldTailByMasking());
+ auto BlockMaskCache = VPlanTransforms::introduceMasksAndLinearize(*Plan);
// ---------------------------------------------------------------------------
// Construct wide recipes and apply predication for original scalar
@@ -8726,7 +8728,8 @@ void LoopVectorizationPlanner::addReductionResultComputation(
auto *RR = dyn_cast<VPReductionRecipe>(OrigExitingVPV->getDefiningRecipe());
if (!PhiR->isInLoop() && CM.foldTailByMasking() &&
(!RR || !RR->isPartialReduction())) {
- VPValue *Cond = RecipeBuilder.getBlockInMask(PhiR->getParent());
+ VPValue *Cond = RecipeBuilder.getBlockInMask(
+ cast<VPBasicBlock>(PhiR->getParent()->getSuccessors().front()));
std::optional<FastMathFlags> FMFs =
PhiTy->isFloatingPointTy()
? std::make_optional(RdxDesc.getFastMathFlags())
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index a6a46e36b397d..d8817e1d481a7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -566,6 +566,11 @@ VPBasicBlock *VPBasicBlock::splitAt(iterator SplitAt) {
auto *SplitBlock = getPlan()->createVPBasicBlock(getName() + ".split");
VPBlockUtils::insertBlockAfter(SplitBlock, this);
+ // If this is the exiting block, make the split the new exiting block.
+ auto *ParentRegion = getParent();
+ if (ParentRegion && ParentRegion->getExiting() == this)
+ ParentRegion->setExiting(SplitBlock);
+
// Finally, move the recipes starting at SplitAt to new block.
for (VPRecipeBase &ToMove :
make_early_inc_range(make_range(SplitAt, this->end())))
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 7ee133545eeb9..97ad127658320 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -958,6 +958,61 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan) {
TopRegion->getEntryBasicBlock()->setName("vector.body");
}
+void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
+ assert(Plan.getExitBlocks().size() == 1 &&
+ "only a single-exit block is supported currently");
+ assert(Plan.getExitBlocks().front()->getSinglePredecessor() ==
+ Plan.getMiddleBlock() &&
+ "the exit block must have middle block as single predecessor");
+
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
+
+ Header->splitAt(Header->getFirstNonPhi());
+
+ // Create the header mask, insert it in the header and branch on it.
+ auto *IV =
+ new VPWidenCanonicalIVRecipe(Header->getParent()->getCanonicalIV());
+ VPBuilder Builder(Header, Header->getFirstNonPhi());
+ Builder.insert(IV);
+ VPValue *BTC = Plan.getOrCreateBackedgeTakenCount();
+ VPValue *HeaderMask = Builder.createICmp(CmpInst::ICMP_ULE, IV, BTC);
+ Builder.createNaryOp(VPInstruction::BranchOnCond, HeaderMask);
+
+ VPBasicBlock *Latch = LoopRegion->getExitingBasicBlock();
+ VPValue *IVInc;
+ [[maybe_unused]] bool TermBranchOnCount =
+ match(Latch->getTerminator(),
+ m_BranchOnCount(m_VPValue(IVInc),
+ m_Specific(&Plan.getVectorTripCount())));
+ assert(TermBranchOnCount &&
+ match(IVInc, m_Add(m_Specific(LoopRegion->getCanonicalIV()),
+ m_Specific(&Plan.getVFxUF()))) &&
+ "Unexpected terminator");
+
+ // Split the latch at the IV update, and branch to it from the header mask.
+ VPBasicBlock *LatchSplit =
+ Latch->splitAt(IVInc->getDefiningRecipe()->getIterator());
+ VPBlockUtils::connectBlocks(Header, LatchSplit);
+
+ // Any extract of the last element must be updated to extract from the last
+ // active lane of the header mask instead (i.e., the lane corresponding to the
+ // last active iteration).
+ Builder.setInsertPoint(Plan.getMiddleBlock()->getTerminator());
+ for (VPRecipeBase &R : *Plan.getMiddleBlock()) {
+ VPValue *Op;
+ if (!match(&R, m_ExtractLastLane(m_ExtractLastPart(m_VPValue(Op)))))
+ continue;
+
+ // Compute the index of the last active lane.
+ VPValue *LastActiveLane =
+ Builder.createNaryOp(VPInstruction::LastActiveLane, HeaderMask);
+ auto *Ext =
+ Builder.createNaryOp(VPInstruction::ExtractLane, {LastActiveLane, Op});
+ R.getVPSingleValue()->replaceAllUsesWith(Ext);
+ }
+}
+
// Likelyhood of bypassing the vectorized loop due to a runtime check block,
// including memory overlap checks block and wrapping/unit-stride checks block.
static constexpr uint32_t CheckBypassWeights[] = {1, 127};
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index f7e7fc29bc203..96bee083875af 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -14,13 +14,11 @@
#include "VPRecipeBuilder.h"
#include "VPlan.h"
#include "VPlanCFG.h"
-#include "VPlanPatternMatch.h"
#include "VPlanTransforms.h"
#include "VPlanUtils.h"
#include "llvm/ADT/PostOrderIterator.h"
using namespace llvm;
-using namespace VPlanPatternMatch;
namespace {
class VPPredicator {
@@ -73,9 +71,6 @@ class VPPredicator {
return EdgeMaskCache.lookup({Src, Dst});
}
- /// Compute and return the mask for the vector loop header block.
- void createHeaderMask(VPBasicBlock *HeaderVPBB, bool FoldTail);
-
/// Compute and return the predicate of \p VPBB, assuming that the header
/// block of the loop is set to True, or to the loop mask when tail folding.
VPValue *createBlockInMask(VPBasicBlock *VPBB);
@@ -156,28 +151,6 @@ VPValue *VPPredicator::createBlockInMask(VPBasicBlock *VPBB) {
return BlockMask;
}
-void VPPredicator::createHeaderMask(VPBasicBlock *HeaderVPBB, bool FoldTail) {
- if (!FoldTail) {
- setBlockInMask(HeaderVPBB, nullptr);
- return;
- }
-
- // Introduce the early-exit compare IV <= BTC to form header block mask.
- // This is used instead of IV < TC because TC may wrap, unlike BTC. Start by
- // constructing the desired canonical IV in the header block as its first
- // non-phi instructions.
-
- auto &Plan = *HeaderVPBB->getPlan();
- auto *IV =
- new VPWidenCanonicalIVRecipe(HeaderVPBB->getParent()->getCanonicalIV());
- Builder.setInsertPoint(HeaderVPBB, HeaderVPBB->getFirstNonPhi());
- Builder.insert(IV);
-
- VPValue *BTC = Plan.getOrCreateBackedgeTakenCount();
- VPValue *BlockMask = Builder.createICmp(CmpInst::ICMP_ULE, IV, BTC);
- setBlockInMask(HeaderVPBB, BlockMask);
-}
-
void VPPredicator::createSwitchEdgeMasks(VPInstruction *SI) {
VPBasicBlock *Src = SI->getParent();
@@ -232,7 +205,8 @@ void VPPredicator::createSwitchEdgeMasks(VPInstruction *SI) {
void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
SmallVector<VPPhi *> Phis;
for (VPRecipeBase &R : VPBB->phis())
- Phis.push_back(cast<VPPhi>(&R));
+ if (auto *PhiR = dyn_cast<VPPhi>(&R))
+ Phis.push_back(PhiR);
for (VPPhi *PhiR : Phis) {
// The non-header Phi is converted into a Blend recipe below,
// so we don't have to worry about the insertion order and we can just use
@@ -261,7 +235,7 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
}
DenseMap<VPBasicBlock *, VPValue *>
-VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan, bool FoldTail) {
+VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
// Scan the body of the loop in a topological order to visit each basic block
// after having visited its predecessor basic blocks.
@@ -272,14 +246,6 @@ VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan, bool FoldTail) {
for (VPBlockBase *VPB : RPOT) {
// Non-outer regions with VPBBs only are supported at the moment.
auto *VPBB = cast<VPBasicBlock>(VPB);
- // Introduce the mask for VPBB, which may introduce needed edge masks, and
- // convert all phi recipes of VPBB to blend recipes unless VPBB is the
- // header.
- if (VPBB == Header) {
- Predicator.createHeaderMask(Header, FoldTail);
- continue;
- }
-
Predicator.createBlockInMask(VPBB);
Predicator.convertPhisToBlends(VPBB);
}
@@ -300,32 +266,5 @@ VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan, bool FoldTail) {
PrevVPBB = VPBB;
}
-
- // If we folded the tail and introduced a header mask, any extract of the
- // last element must be updated to extract from the last active lane of the
- // header mask instead (i.e., the lane corresponding to the last active
- // iteration).
- if (FoldTail) {
- assert(Plan.getExitBlocks().size() == 1 &&
- "only a single-exit block is supported currently");
- assert(Plan.getExitBlocks().front()->getSinglePredecessor() ==
- Plan.getMiddleBlock() &&
- "the exit block must have middle block as single predecessor");
-
- VPBuilder B(Plan.getMiddleBlock()->getTerminator());
- for (VPRecipeBase &R : *Plan.getMiddleBlock()) {
- VPValue *Op;
- if (!match(&R, m_ExtractLastLane(m_ExtractLastPart(m_VPValue(Op)))))
- continue;
-
- // Compute the index of the last active lane.
- VPValue *HeaderMask = Predicator.getBlockInMask(Header);
- VPValue *LastActiveLane =
- B.createNaryOp(VPInstruction::LastActiveLane, HeaderMask);
- auto *Ext =
- B.createNaryOp(VPInstruction::ExtractLane, {LastActiveLane, Op});
- R.getVPSingleValue()->replaceAllUsesWith(Ext);
- }
- }
return Predicator.getBlockMaskCache();
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index e0d09a099647a..e00c94558bf7a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -413,14 +413,44 @@ struct VPlanTransforms {
static void narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
TypeSize VectorRegWidth);
+ /// Adapts the vector loop region for tail folding by introducing a header
+ /// mask and predicating the region:
+ ///
+ /// Vector loop region before:
+ /// +-------------------------------------------+
+ /// |%iv = ... |
+ /// |... |
+ /// |%iv.next = add %iv, vfxuf |
+ /// |branch-on-count %iv.next, vector-trip-count|
+ /// +-------------------------------------------+
+ ///
+ /// Vector loop region after:
+ /// +-------------------------------------------+
+ /// |%iv = ... |
+ /// |%wide.iv = widen-canonical-iv ... |
+ /// |%header-mask = icmp ult %wide.iv, BTC |
+ /// |branch-on-cond %header-mask |---+
+ /// +-------------------------------------------+ |
+ /// | |
+ /// v |
+ /// +-------------------------------------------+ |
+ /// | ... | |
+ /// +-------------------------------------------+ |
+ /// | |
+ /// v |
+ /// +-------------------------------------------+ |
+ /// |%iv.next = add %iv, vfxuf |<--+
+ /// |branch-on-count %iv.next, vector-trip-count|
+ /// +-------------------------------------------+
+ ///
+ /// Any VPInstruction::ExtractLastLanes are also updated to extract from the
+ /// last active lane of the header mask.
+ static void foldTailByMasking(VPlan &Plan);
+
/// Predicate and linearize the control-flow in the only loop region of
- /// \p Plan. If \p FoldTail is true, create a mask guarding the loop
- /// header, otherwise use all-true for the header mask. Masks for blocks are
- /// added to a block-to-mask map which is returned in order to be used later
- /// for wide recipe construction. This argument is temporary and will be
- /// removed in the future.
+ /// \p Plan.
static DenseMap<VPBasicBlock *, VPValue *>
- introduceMasksAndLinearize(VPlan &Plan, bool FoldTail);
+ introduceMasksAndLinearize(VPlan &Plan);
/// Add branch weight metadata, if the \p Plan's middle block is terminated by
/// a BranchOnCond recipe.
>From ed18c28e87494515bf398919b68fbb0a0527123d Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 22 Jan 2026 01:13:20 +0800
Subject: [PATCH 2/9] Don't use branch-on-cond, explicitly pass mask in
densemap to VPPredicator
This preserves SSA
---
.../Transforms/Vectorize/LoopVectorize.cpp | 6 ++-
.../Vectorize/VPlanConstruction.cpp | 23 ++------
.../Transforms/Vectorize/VPlanPredicator.cpp | 17 ++++--
.../Transforms/Vectorize/VPlanTransforms.h | 53 ++++++-------------
4 files changed, 40 insertions(+), 59 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1dcd33828445a..26462483b2059 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8463,13 +8463,15 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
InterleaveGroups.insert(IG);
}
+ DenseMap<VPBasicBlock *, VPValue *> SuccessorMasks;
if (CM.foldTailByMasking())
- VPlanTransforms::foldTailByMasking(*Plan);
+ VPlanTransforms::foldTailByMasking(*Plan, SuccessorMasks);
// ---------------------------------------------------------------------------
// Predicate and linearize the top-level loop region.
// ---------------------------------------------------------------------------
- auto BlockMaskCache = VPlanTransforms::introduceMasksAndLinearize(*Plan);
+ auto BlockMaskCache =
+ VPlanTransforms::introduceMasksAndLinearize(*Plan, SuccessorMasks);
// ---------------------------------------------------------------------------
// Construct wide recipes and apply predication for original scalar
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 97ad127658320..8daef2ba03825 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -958,7 +958,8 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan) {
TopRegion->getEntryBasicBlock()->setName("vector.body");
}
-void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
+void VPlanTransforms::foldTailByMasking(
+ VPlan &Plan, DenseMap<VPBasicBlock *, VPValue *> &SuccessorMasks) {
assert(Plan.getExitBlocks().size() == 1 &&
"only a single-exit block is supported currently");
assert(Plan.getExitBlocks().front()->getSinglePredecessor() ==
@@ -970,30 +971,16 @@ void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
Header->splitAt(Header->getFirstNonPhi());
- // Create the header mask, insert it in the header and branch on it.
+ // Create the header mask and insert it in the header.
auto *IV =
new VPWidenCanonicalIVRecipe(Header->getParent()->getCanonicalIV());
VPBuilder Builder(Header, Header->getFirstNonPhi());
Builder.insert(IV);
VPValue *BTC = Plan.getOrCreateBackedgeTakenCount();
VPValue *HeaderMask = Builder.createICmp(CmpInst::ICMP_ULE, IV, BTC);
- Builder.createNaryOp(VPInstruction::BranchOnCond, HeaderMask);
-
- VPBasicBlock *Latch = LoopRegion->getExitingBasicBlock();
- VPValue *IVInc;
- [[maybe_unused]] bool TermBranchOnCount =
- match(Latch->getTerminator(),
- m_BranchOnCount(m_VPValue(IVInc),
- m_Specific(&Plan.getVectorTripCount())));
- assert(TermBranchOnCount &&
- match(IVInc, m_Add(m_Specific(LoopRegion->getCanonicalIV()),
- m_Specific(&Plan.getVFxUF()))) &&
- "Unexpected terminator");
- // Split the latch at the IV update, and branch to it from the header mask.
- VPBasicBlock *LatchSplit =
- Latch->splitAt(IVInc->getDefiningRecipe()->getIterator());
- VPBlockUtils::connectBlocks(Header, LatchSplit);
+ // Predicate everything after the header mask.
+ SuccessorMasks[Header] = HeaderMask;
// Any extract of the last element must be updated to extract from the last
// active lane of the header mask instead (i.e., the lane corresponding to the
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 96bee083875af..251a3afa4634d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -35,6 +35,10 @@ class VPPredicator {
BlockMaskCacheTy BlockMaskCache;
+ /// A map of blocks to masks that should be applied to each of its successors'
+ /// edges.
+ const BlockMaskCacheTy SuccessorMasks;
+
/// Create an edge mask for every destination of cases and/or default.
void createSwitchEdgeMasks(VPInstruction *SI);
@@ -61,6 +65,9 @@ class VPPredicator {
}
public:
+ VPPredicator(const BlockMaskCacheTy &SuccessorMasks)
+ : SuccessorMasks(SuccessorMasks) {}
+
/// Returns the *entry* mask for \p VPBB.
VPValue *getBlockInMask(VPBasicBlock *VPBB) const {
return BlockMaskCache.lookup(VPBB);
@@ -92,6 +99,10 @@ VPValue *VPPredicator::createEdgeMask(VPBasicBlock *Src, VPBasicBlock *Dst) {
VPValue *SrcMask = getBlockInMask(Src);
+ if (VPValue *SuccessorMask = SuccessorMasks.lookup(Src))
+ SrcMask = SrcMask ? Builder.createLogicalAnd(SrcMask, SuccessorMask)
+ : SuccessorMask;
+
// If there's a single successor, there's no terminator recipe.
if (Src->getNumSuccessors() == 1)
return setEdgeMask(Src, Dst, SrcMask);
@@ -234,15 +245,15 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
}
}
-DenseMap<VPBasicBlock *, VPValue *>
-VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
+DenseMap<VPBasicBlock *, VPValue *> VPlanTransforms::introduceMasksAndLinearize(
+ VPlan &Plan, const DenseMap<VPBasicBlock *, VPValue *> &SuccessorMasks) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
// Scan the body of the loop in a topological order to visit each basic block
// after having visited its predecessor basic blocks.
VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
Header);
- VPPredicator Predicator;
+ VPPredicator Predicator(SuccessorMasks);
for (VPBlockBase *VPB : RPOT) {
// Non-outer regions with VPBBs only are supported at the moment.
auto *VPBB = cast<VPBasicBlock>(VPB);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index e00c94558bf7a..cf0147e5a223c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -413,44 +413,25 @@ struct VPlanTransforms {
static void narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
TypeSize VectorRegWidth);
- /// Adapts the vector loop region for tail folding by introducing a header
- /// mask and predicating the region:
+ /// Adapts the vector loop region for tail folding by inserting a header mask
+ /// after the first non-phi in the header:
///
- /// Vector loop region before:
- /// +-------------------------------------------+
- /// |%iv = ... |
- /// |... |
- /// |%iv.next = add %iv, vfxuf |
- /// |branch-on-count %iv.next, vector-trip-count|
- /// +-------------------------------------------+
+ /// %wide.iv = widen-canonical-iv ...
+ /// %header-mask = icmp ult %wide.iv, BTC
///
- /// Vector loop region after:
- /// +-------------------------------------------+
- /// |%iv = ... |
- /// |%wide.iv = widen-canonical-iv ... |
- /// |%header-mask = icmp ult %wide.iv, BTC |
- /// |branch-on-cond %header-mask |---+
- /// +-------------------------------------------+ |
- /// | |
- /// v |
- /// +-------------------------------------------+ |
- /// | ... | |
- /// +-------------------------------------------+ |
- /// | |
- /// v |
- /// +-------------------------------------------+ |
- /// |%iv.next = add %iv, vfxuf |<--+
- /// |branch-on-count %iv.next, vector-trip-count|
- /// +-------------------------------------------+
- ///
- /// Any VPInstruction::ExtractLastLanes are also updated to extract from the
- /// last active lane of the header mask.
- static void foldTailByMasking(VPlan &Plan);
-
- /// Predicate and linearize the control-flow in the only loop region of
- /// \p Plan.
- static DenseMap<VPBasicBlock *, VPValue *>
- introduceMasksAndLinearize(VPlan &Plan);
+ /// The header is then split at the header mask, and everything afterwards is
+ /// predicated by adding the header mask to \p SuccessorMasks. Any
+ /// VPInstruction::ExtractLastLanes are also updated to extract from the last
+ /// active lane of the header mask.
+ static void
+ foldTailByMasking(VPlan &Plan,
+ DenseMap<VPBasicBlock *, VPValue *> &SuccessorMasks);
+
+ /// Predicate and linearize the control-flow in the only loop region of \p
+ /// Plan. Masks for blocks are added to a block-to-mask map which is returned
+ /// in order to be used later for wide recipe construction.
+ static DenseMap<VPBasicBlock *, VPValue *> introduceMasksAndLinearize(
+ VPlan &Plan, const DenseMap<VPBasicBlock *, VPValue *> &SuccessorMasks);
/// Add branch weight metadata, if the \p Plan's middle block is terminated by
/// a BranchOnCond recipe.
>From 81332b3fa28e95819fe9d67310a45409b15a833e Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 22 Jan 2026 18:42:48 +0800
Subject: [PATCH 3/9] Don't pass state, use VPInstruction::PredicateSuccessors
---
.../Transforms/Vectorize/LoopVectorize.cpp | 6 ++--
llvm/lib/Transforms/Vectorize/VPlan.h | 4 +++
.../Vectorize/VPlanConstruction.cpp | 6 ++--
.../Transforms/Vectorize/VPlanPredicator.cpp | 28 +++++++++----------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 1 +
.../Transforms/Vectorize/VPlanTransforms.h | 12 ++++----
6 files changed, 28 insertions(+), 29 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 26462483b2059..1dcd33828445a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8463,15 +8463,13 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
InterleaveGroups.insert(IG);
}
- DenseMap<VPBasicBlock *, VPValue *> SuccessorMasks;
if (CM.foldTailByMasking())
- VPlanTransforms::foldTailByMasking(*Plan, SuccessorMasks);
+ VPlanTransforms::foldTailByMasking(*Plan);
// ---------------------------------------------------------------------------
// Predicate and linearize the top-level loop region.
// ---------------------------------------------------------------------------
- auto BlockMaskCache =
- VPlanTransforms::introduceMasksAndLinearize(*Plan, SuccessorMasks);
+ auto BlockMaskCache = VPlanTransforms::introduceMasksAndLinearize(*Plan);
// ---------------------------------------------------------------------------
// Construct wide recipes and apply predication for original scalar
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 1d250322ccf63..33884726e0d80 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1193,6 +1193,10 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// active (non-zero) lane in the mask (second operand), or if no lanes
/// were active in the mask, returns the default value (third operand).
ExtractLastActive,
+ /// A transient terminator that indicates any successors of the block should
+ /// be predicated with the mask from the first operand. Should be removed
+ /// after introduceMasksAndLinearize.
+ PredicateSuccessors,
/// Returns the value for vscale.
VScale,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 8daef2ba03825..89507b68d4b0c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -958,8 +958,7 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan) {
TopRegion->getEntryBasicBlock()->setName("vector.body");
}
-void VPlanTransforms::foldTailByMasking(
- VPlan &Plan, DenseMap<VPBasicBlock *, VPValue *> &SuccessorMasks) {
+void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
assert(Plan.getExitBlocks().size() == 1 &&
"only a single-exit block is supported currently");
assert(Plan.getExitBlocks().front()->getSinglePredecessor() ==
@@ -980,7 +979,8 @@ void VPlanTransforms::foldTailByMasking(
VPValue *HeaderMask = Builder.createICmp(CmpInst::ICMP_ULE, IV, BTC);
// Predicate everything after the header mask.
- SuccessorMasks[Header] = HeaderMask;
+ Builder.setInsertPoint(Header, Header->end());
+ Builder.createNaryOp(VPInstruction::PredicateSuccessors, HeaderMask);
// Any extract of the last element must be updated to extract from the last
// active lane of the header mask instead (i.e., the lane corresponding to the
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 251a3afa4634d..3c51bdea5cb25 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -14,11 +14,13 @@
#include "VPRecipeBuilder.h"
#include "VPlan.h"
#include "VPlanCFG.h"
+#include "VPlanPatternMatch.h"
#include "VPlanTransforms.h"
#include "VPlanUtils.h"
#include "llvm/ADT/PostOrderIterator.h"
using namespace llvm;
+using namespace VPlanPatternMatch;
namespace {
class VPPredicator {
@@ -35,10 +37,6 @@ class VPPredicator {
BlockMaskCacheTy BlockMaskCache;
- /// A map of blocks to masks that should be applied to each of its successors'
- /// edges.
- const BlockMaskCacheTy SuccessorMasks;
-
/// Create an edge mask for every destination of cases and/or default.
void createSwitchEdgeMasks(VPInstruction *SI);
@@ -65,9 +63,6 @@ class VPPredicator {
}
public:
- VPPredicator(const BlockMaskCacheTy &SuccessorMasks)
- : SuccessorMasks(SuccessorMasks) {}
-
/// Returns the *entry* mask for \p VPBB.
VPValue *getBlockInMask(VPBasicBlock *VPBB) const {
return BlockMaskCache.lookup(VPBB);
@@ -78,8 +73,7 @@ class VPPredicator {
return EdgeMaskCache.lookup({Src, Dst});
}
- /// Compute and return the predicate of \p VPBB, assuming that the header
- /// block of the loop is set to True, or to the loop mask when tail folding.
+ /// Compute and return the predicate of \p VPBB.
VPValue *createBlockInMask(VPBasicBlock *VPBB);
/// Convert phi recipes in \p VPBB to VPBlendRecipes.
@@ -99,9 +93,13 @@ VPValue *VPPredicator::createEdgeMask(VPBasicBlock *Src, VPBasicBlock *Dst) {
VPValue *SrcMask = getBlockInMask(Src);
- if (VPValue *SuccessorMask = SuccessorMasks.lookup(Src))
- SrcMask = SrcMask ? Builder.createLogicalAnd(SrcMask, SuccessorMask)
- : SuccessorMask;
+ VPValue *SuccPred;
+ if (!Src->empty() &&
+ match(&Src->back(), m_VPInstruction<VPInstruction::PredicateSuccessors>(
+ m_VPValue(SuccPred)))) {
+ Src->back().eraseFromParent();
+ SrcMask = SrcMask ? Builder.createLogicalAnd(SrcMask, SuccPred) : SuccPred;
+ }
// If there's a single successor, there's no terminator recipe.
if (Src->getNumSuccessors() == 1)
@@ -245,15 +243,15 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
}
}
-DenseMap<VPBasicBlock *, VPValue *> VPlanTransforms::introduceMasksAndLinearize(
- VPlan &Plan, const DenseMap<VPBasicBlock *, VPValue *> &SuccessorMasks) {
+DenseMap<VPBasicBlock *, VPValue *>
+VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
// Scan the body of the loop in a topological order to visit each basic block
// after having visited its predecessor basic blocks.
VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
Header);
- VPPredicator Predicator(SuccessorMasks);
+ VPPredicator Predicator;
for (VPBlockBase *VPB : RPOT) {
// Non-outer regions with VPBBs only are supported at the moment.
auto *VPBB = cast<VPBasicBlock>(VPB);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 0c6100c04f785..4a43255cd7244 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -454,6 +454,7 @@ unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) {
case VPInstruction::ExtractLastPart:
case VPInstruction::ExtractPenultimateElement:
case VPInstruction::Not:
+ case VPInstruction::PredicateSuccessors:
case VPInstruction::ResumeForEpilogue:
case VPInstruction::Reverse:
case VPInstruction::Unpack:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index cf0147e5a223c..740420f9e3ad9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -419,19 +419,17 @@ struct VPlanTransforms {
/// %wide.iv = widen-canonical-iv ...
/// %header-mask = icmp ult %wide.iv, BTC
///
- /// The header is then split at the header mask, and everything afterwards is
- /// predicated by adding the header mask to \p SuccessorMasks. Any
+ /// The header is then split at the header mask and successors are predicated
+ /// with VPInstruction::PredicateSuccessors. Any
/// VPInstruction::ExtractLastLanes are also updated to extract from the last
/// active lane of the header mask.
- static void
- foldTailByMasking(VPlan &Plan,
- DenseMap<VPBasicBlock *, VPValue *> &SuccessorMasks);
+ static void foldTailByMasking(VPlan &Plan);
/// Predicate and linearize the control-flow in the only loop region of \p
/// Plan. Masks for blocks are added to a block-to-mask map which is returned
/// in order to be used later for wide recipe construction.
- static DenseMap<VPBasicBlock *, VPValue *> introduceMasksAndLinearize(
- VPlan &Plan, const DenseMap<VPBasicBlock *, VPValue *> &SuccessorMasks);
+ static DenseMap<VPBasicBlock *, VPValue *>
+ introduceMasksAndLinearize(VPlan &Plan);
/// Add branch weight metadata, if the \p Plan's middle block is terminated by
/// a BranchOnCond recipe.
>From 5475f482929c1b2a325b9295b85878785d5ba281 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 23 Jan 2026 19:43:20 +0800
Subject: [PATCH 4/9] Revert "Don't pass state, use
VPInstruction::PredicateSuccessors"
This reverts commit 81332b3fa28e95819fe9d67310a45409b15a833e.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 6 ++--
llvm/lib/Transforms/Vectorize/VPlan.h | 4 ---
.../Vectorize/VPlanConstruction.cpp | 6 ++--
.../Transforms/Vectorize/VPlanPredicator.cpp | 28 ++++++++++---------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 1 -
.../Transforms/Vectorize/VPlanTransforms.h | 12 ++++----
6 files changed, 29 insertions(+), 28 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1dcd33828445a..26462483b2059 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8463,13 +8463,15 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
InterleaveGroups.insert(IG);
}
+ DenseMap<VPBasicBlock *, VPValue *> SuccessorMasks;
if (CM.foldTailByMasking())
- VPlanTransforms::foldTailByMasking(*Plan);
+ VPlanTransforms::foldTailByMasking(*Plan, SuccessorMasks);
// ---------------------------------------------------------------------------
// Predicate and linearize the top-level loop region.
// ---------------------------------------------------------------------------
- auto BlockMaskCache = VPlanTransforms::introduceMasksAndLinearize(*Plan);
+ auto BlockMaskCache =
+ VPlanTransforms::introduceMasksAndLinearize(*Plan, SuccessorMasks);
// ---------------------------------------------------------------------------
// Construct wide recipes and apply predication for original scalar
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 33884726e0d80..1d250322ccf63 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1193,10 +1193,6 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// active (non-zero) lane in the mask (second operand), or if no lanes
/// were active in the mask, returns the default value (third operand).
ExtractLastActive,
- /// A transient terminator that indicates any successors of the block should
- /// be predicated with the mask from the first operand. Should be removed
- /// after introduceMasksAndLinearize.
- PredicateSuccessors,
/// Returns the value for vscale.
VScale,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 89507b68d4b0c..8daef2ba03825 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -958,7 +958,8 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan) {
TopRegion->getEntryBasicBlock()->setName("vector.body");
}
-void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
+void VPlanTransforms::foldTailByMasking(
+ VPlan &Plan, DenseMap<VPBasicBlock *, VPValue *> &SuccessorMasks) {
assert(Plan.getExitBlocks().size() == 1 &&
"only a single-exit block is supported currently");
assert(Plan.getExitBlocks().front()->getSinglePredecessor() ==
@@ -979,8 +980,7 @@ void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
VPValue *HeaderMask = Builder.createICmp(CmpInst::ICMP_ULE, IV, BTC);
// Predicate everything after the header mask.
- Builder.setInsertPoint(Header, Header->end());
- Builder.createNaryOp(VPInstruction::PredicateSuccessors, HeaderMask);
+ SuccessorMasks[Header] = HeaderMask;
// Any extract of the last element must be updated to extract from the last
// active lane of the header mask instead (i.e., the lane corresponding to the
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 3c51bdea5cb25..251a3afa4634d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -14,13 +14,11 @@
#include "VPRecipeBuilder.h"
#include "VPlan.h"
#include "VPlanCFG.h"
-#include "VPlanPatternMatch.h"
#include "VPlanTransforms.h"
#include "VPlanUtils.h"
#include "llvm/ADT/PostOrderIterator.h"
using namespace llvm;
-using namespace VPlanPatternMatch;
namespace {
class VPPredicator {
@@ -37,6 +35,10 @@ class VPPredicator {
BlockMaskCacheTy BlockMaskCache;
+ /// A map of blocks to masks that should be applied to each of its successors'
+ /// edges.
+ const BlockMaskCacheTy SuccessorMasks;
+
/// Create an edge mask for every destination of cases and/or default.
void createSwitchEdgeMasks(VPInstruction *SI);
@@ -63,6 +65,9 @@ class VPPredicator {
}
public:
+ VPPredicator(const BlockMaskCacheTy &SuccessorMasks)
+ : SuccessorMasks(SuccessorMasks) {}
+
/// Returns the *entry* mask for \p VPBB.
VPValue *getBlockInMask(VPBasicBlock *VPBB) const {
return BlockMaskCache.lookup(VPBB);
@@ -73,7 +78,8 @@ class VPPredicator {
return EdgeMaskCache.lookup({Src, Dst});
}
- /// Compute and return the predicate of \p VPBB.
+ /// Compute and return the predicate of \p VPBB, assuming that the header
+ /// block of the loop is set to True, or to the loop mask when tail folding.
VPValue *createBlockInMask(VPBasicBlock *VPBB);
/// Convert phi recipes in \p VPBB to VPBlendRecipes.
@@ -93,13 +99,9 @@ VPValue *VPPredicator::createEdgeMask(VPBasicBlock *Src, VPBasicBlock *Dst) {
VPValue *SrcMask = getBlockInMask(Src);
- VPValue *SuccPred;
- if (!Src->empty() &&
- match(&Src->back(), m_VPInstruction<VPInstruction::PredicateSuccessors>(
- m_VPValue(SuccPred)))) {
- Src->back().eraseFromParent();
- SrcMask = SrcMask ? Builder.createLogicalAnd(SrcMask, SuccPred) : SuccPred;
- }
+ if (VPValue *SuccessorMask = SuccessorMasks.lookup(Src))
+ SrcMask = SrcMask ? Builder.createLogicalAnd(SrcMask, SuccessorMask)
+ : SuccessorMask;
// If there's a single successor, there's no terminator recipe.
if (Src->getNumSuccessors() == 1)
@@ -243,15 +245,15 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
}
}
-DenseMap<VPBasicBlock *, VPValue *>
-VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
+DenseMap<VPBasicBlock *, VPValue *> VPlanTransforms::introduceMasksAndLinearize(
+ VPlan &Plan, const DenseMap<VPBasicBlock *, VPValue *> &SuccessorMasks) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
// Scan the body of the loop in a topological order to visit each basic block
// after having visited its predecessor basic blocks.
VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
Header);
- VPPredicator Predicator;
+ VPPredicator Predicator(SuccessorMasks);
for (VPBlockBase *VPB : RPOT) {
// Non-outer regions with VPBBs only are supported at the moment.
auto *VPBB = cast<VPBasicBlock>(VPB);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 4a43255cd7244..0c6100c04f785 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -454,7 +454,6 @@ unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) {
case VPInstruction::ExtractLastPart:
case VPInstruction::ExtractPenultimateElement:
case VPInstruction::Not:
- case VPInstruction::PredicateSuccessors:
case VPInstruction::ResumeForEpilogue:
case VPInstruction::Reverse:
case VPInstruction::Unpack:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 740420f9e3ad9..cf0147e5a223c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -419,17 +419,19 @@ struct VPlanTransforms {
/// %wide.iv = widen-canonical-iv ...
/// %header-mask = icmp ult %wide.iv, BTC
///
- /// The header is then split at the header mask and successors are predicated
- /// with VPInstruction::PredicateSuccessors. Any
+ /// The header is then split at the header mask, and everything afterwards is
+ /// predicated by adding the header mask to \p SuccessorMasks. Any
/// VPInstruction::ExtractLastLanes are also updated to extract from the last
/// active lane of the header mask.
- static void foldTailByMasking(VPlan &Plan);
+ static void
+ foldTailByMasking(VPlan &Plan,
+ DenseMap<VPBasicBlock *, VPValue *> &SuccessorMasks);
/// Predicate and linearize the control-flow in the only loop region of \p
/// Plan. Masks for blocks are added to a block-to-mask map which is returned
/// in order to be used later for wide recipe construction.
- static DenseMap<VPBasicBlock *, VPValue *>
- introduceMasksAndLinearize(VPlan &Plan);
+ static DenseMap<VPBasicBlock *, VPValue *> introduceMasksAndLinearize(
+ VPlan &Plan, const DenseMap<VPBasicBlock *, VPValue *> &SuccessorMasks);
/// Add branch weight metadata, if the \p Plan's middle block is terminated by
/// a BranchOnCond recipe.
>From d7bb1118b3536973310c829a15049d8866c838c3 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Fri, 23 Jan 2026 19:43:23 +0800
Subject: [PATCH 5/9] Revert "Don't use branch-on-cond, explicitly pass mask in
densemap to VPPredicator"
This reverts commit ed18c28e87494515bf398919b68fbb0a0527123d.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 6 +--
.../Vectorize/VPlanConstruction.cpp | 23 ++++++--
.../Transforms/Vectorize/VPlanPredicator.cpp | 17 ++----
.../Transforms/Vectorize/VPlanTransforms.h | 53 +++++++++++++------
4 files changed, 59 insertions(+), 40 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 26462483b2059..1dcd33828445a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8463,15 +8463,13 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
InterleaveGroups.insert(IG);
}
- DenseMap<VPBasicBlock *, VPValue *> SuccessorMasks;
if (CM.foldTailByMasking())
- VPlanTransforms::foldTailByMasking(*Plan, SuccessorMasks);
+ VPlanTransforms::foldTailByMasking(*Plan);
// ---------------------------------------------------------------------------
// Predicate and linearize the top-level loop region.
// ---------------------------------------------------------------------------
- auto BlockMaskCache =
- VPlanTransforms::introduceMasksAndLinearize(*Plan, SuccessorMasks);
+ auto BlockMaskCache = VPlanTransforms::introduceMasksAndLinearize(*Plan);
// ---------------------------------------------------------------------------
// Construct wide recipes and apply predication for original scalar
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 8daef2ba03825..97ad127658320 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -958,8 +958,7 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan) {
TopRegion->getEntryBasicBlock()->setName("vector.body");
}
-void VPlanTransforms::foldTailByMasking(
- VPlan &Plan, DenseMap<VPBasicBlock *, VPValue *> &SuccessorMasks) {
+void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
assert(Plan.getExitBlocks().size() == 1 &&
"only a single-exit block is supported currently");
assert(Plan.getExitBlocks().front()->getSinglePredecessor() ==
@@ -971,16 +970,30 @@ void VPlanTransforms::foldTailByMasking(
Header->splitAt(Header->getFirstNonPhi());
- // Create the header mask and insert it in the header.
+ // Create the header mask, insert it in the header and branch on it.
auto *IV =
new VPWidenCanonicalIVRecipe(Header->getParent()->getCanonicalIV());
VPBuilder Builder(Header, Header->getFirstNonPhi());
Builder.insert(IV);
VPValue *BTC = Plan.getOrCreateBackedgeTakenCount();
VPValue *HeaderMask = Builder.createICmp(CmpInst::ICMP_ULE, IV, BTC);
+ Builder.createNaryOp(VPInstruction::BranchOnCond, HeaderMask);
+
+ VPBasicBlock *Latch = LoopRegion->getExitingBasicBlock();
+ VPValue *IVInc;
+ [[maybe_unused]] bool TermBranchOnCount =
+ match(Latch->getTerminator(),
+ m_BranchOnCount(m_VPValue(IVInc),
+ m_Specific(&Plan.getVectorTripCount())));
+ assert(TermBranchOnCount &&
+ match(IVInc, m_Add(m_Specific(LoopRegion->getCanonicalIV()),
+ m_Specific(&Plan.getVFxUF()))) &&
+ "Unexpected terminator");
- // Predicate everything after the header mask.
- SuccessorMasks[Header] = HeaderMask;
+ // Split the latch at the IV update, and branch to it from the header mask.
+ VPBasicBlock *LatchSplit =
+ Latch->splitAt(IVInc->getDefiningRecipe()->getIterator());
+ VPBlockUtils::connectBlocks(Header, LatchSplit);
// Any extract of the last element must be updated to extract from the last
// active lane of the header mask instead (i.e., the lane corresponding to the
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 251a3afa4634d..96bee083875af 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -35,10 +35,6 @@ class VPPredicator {
BlockMaskCacheTy BlockMaskCache;
- /// A map of blocks to masks that should be applied to each of its successors'
- /// edges.
- const BlockMaskCacheTy SuccessorMasks;
-
/// Create an edge mask for every destination of cases and/or default.
void createSwitchEdgeMasks(VPInstruction *SI);
@@ -65,9 +61,6 @@ class VPPredicator {
}
public:
- VPPredicator(const BlockMaskCacheTy &SuccessorMasks)
- : SuccessorMasks(SuccessorMasks) {}
-
/// Returns the *entry* mask for \p VPBB.
VPValue *getBlockInMask(VPBasicBlock *VPBB) const {
return BlockMaskCache.lookup(VPBB);
@@ -99,10 +92,6 @@ VPValue *VPPredicator::createEdgeMask(VPBasicBlock *Src, VPBasicBlock *Dst) {
VPValue *SrcMask = getBlockInMask(Src);
- if (VPValue *SuccessorMask = SuccessorMasks.lookup(Src))
- SrcMask = SrcMask ? Builder.createLogicalAnd(SrcMask, SuccessorMask)
- : SuccessorMask;
-
// If there's a single successor, there's no terminator recipe.
if (Src->getNumSuccessors() == 1)
return setEdgeMask(Src, Dst, SrcMask);
@@ -245,15 +234,15 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
}
}
-DenseMap<VPBasicBlock *, VPValue *> VPlanTransforms::introduceMasksAndLinearize(
- VPlan &Plan, const DenseMap<VPBasicBlock *, VPValue *> &SuccessorMasks) {
+DenseMap<VPBasicBlock *, VPValue *>
+VPlanTransforms::introduceMasksAndLinearize(VPlan &Plan) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
// Scan the body of the loop in a topological order to visit each basic block
// after having visited its predecessor basic blocks.
VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
Header);
- VPPredicator Predicator(SuccessorMasks);
+ VPPredicator Predicator;
for (VPBlockBase *VPB : RPOT) {
// Non-outer regions with VPBBs only are supported at the moment.
auto *VPBB = cast<VPBasicBlock>(VPB);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index cf0147e5a223c..e00c94558bf7a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -413,25 +413,44 @@ struct VPlanTransforms {
static void narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
TypeSize VectorRegWidth);
- /// Adapts the vector loop region for tail folding by inserting a header mask
- /// after the first non-phi in the header:
+ /// Adapts the vector loop region for tail folding by introducing a header
+ /// mask and predicating the region:
///
- /// %wide.iv = widen-canonical-iv ...
- /// %header-mask = icmp ult %wide.iv, BTC
+ /// Vector loop region before:
+ /// +-------------------------------------------+
+ /// |%iv = ... |
+ /// |... |
+ /// |%iv.next = add %iv, vfxuf |
+ /// |branch-on-count %iv.next, vector-trip-count|
+ /// +-------------------------------------------+
///
- /// The header is then split at the header mask, and everything afterwards is
- /// predicated by adding the header mask to \p SuccessorMasks. Any
- /// VPInstruction::ExtractLastLanes are also updated to extract from the last
- /// active lane of the header mask.
- static void
- foldTailByMasking(VPlan &Plan,
- DenseMap<VPBasicBlock *, VPValue *> &SuccessorMasks);
-
- /// Predicate and linearize the control-flow in the only loop region of \p
- /// Plan. Masks for blocks are added to a block-to-mask map which is returned
- /// in order to be used later for wide recipe construction.
- static DenseMap<VPBasicBlock *, VPValue *> introduceMasksAndLinearize(
- VPlan &Plan, const DenseMap<VPBasicBlock *, VPValue *> &SuccessorMasks);
+ /// Vector loop region after:
+ /// +-------------------------------------------+
+ /// |%iv = ... |
+ /// |%wide.iv = widen-canonical-iv ... |
+ /// |%header-mask = icmp ult %wide.iv, BTC |
+ /// |branch-on-cond %header-mask |---+
+ /// +-------------------------------------------+ |
+ /// | |
+ /// v |
+ /// +-------------------------------------------+ |
+ /// | ... | |
+ /// +-------------------------------------------+ |
+ /// | |
+ /// v |
+ /// +-------------------------------------------+ |
+ /// |%iv.next = add %iv, vfxuf |<--+
+ /// |branch-on-count %iv.next, vector-trip-count|
+ /// +-------------------------------------------+
+ ///
+ /// Any VPInstruction::ExtractLastLanes are also updated to extract from the
+ /// last active lane of the header mask.
+ static void foldTailByMasking(VPlan &Plan);
+
+ /// Predicate and linearize the control-flow in the only loop region of
+ /// \p Plan.
+ static DenseMap<VPBasicBlock *, VPValue *>
+ introduceMasksAndLinearize(VPlan &Plan);
/// Add branch weight metadata, if the \p Plan's middle block is terminated by
/// a BranchOnCond recipe.
>From b7842de37323a15a8b50a99ba598f65e1283c4e8 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Sat, 24 Jan 2026 00:03:16 +0800
Subject: [PATCH 6/9] Maintain SSA
---
.../Vectorize/VPlanConstruction.cpp | 27 ++++++++++++++++++-
.../Transforms/Vectorize/VPlanPredicator.cpp | 8 +++++-
llvm/lib/Transforms/Vectorize/VPlanValue.h | 8 ++++++
3 files changed, 41 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 97ad127658320..10bec7c91af1e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -968,7 +968,7 @@ void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
- Header->splitAt(Header->getFirstNonPhi());
+ VPBasicBlock *HeaderSplit = Header->splitAt(Header->getFirstNonPhi());
// Create the header mask, insert it in the header and branch on it.
auto *IV =
@@ -995,6 +995,31 @@ void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
Latch->splitAt(IVInc->getDefiningRecipe()->getIterator());
VPBlockUtils::connectBlocks(Header, LatchSplit);
+ // Insert phis for any values used outside of the predicated body.
+ auto NeedsPhi = [&Header, &Plan](VPUser *U) {
+ auto *UR = cast<VPRecipeBase>(U);
+ return UR->getParent() == Header ||
+ UR->getRegion() != Plan.getVectorLoopRegion();
+ };
+
+ VPTypeAnalysis TypeInfo(Plan);
+ Builder.setInsertPoint(LatchSplit, LatchSplit->begin());
+ for (VPBlockBase *VPB : vp_depth_first_shallow(HeaderSplit)) {
+ auto *VPBB = cast<VPBasicBlock>(VPB);
+ if (VPBB == LatchSplit)
+ continue;
+ for (VPRecipeBase &R : *VPBB) {
+ for (VPValue *V : R.definedValues()) {
+ if (!any_of(V->users(), NeedsPhi))
+ continue;
+ VPValue *Poison = Plan.getOrAddLiveIn(
+ PoisonValue::get(V->getUnderlyingValue()->getType()));
+ VPValue *Phi = Builder.createScalarPhi({V, Poison}, {});
+ V->replaceUsesWithIf(Phi, NeedsPhi);
+ }
+ }
+ }
+
// Any extract of the last element must be updated to extract from the last
// active lane of the header mask instead (i.e., the lane corresponding to the
// last active iteration).
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 96bee083875af..b62e06025bbcb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -214,7 +214,13 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
// be duplications since this is a simple recursive scan, but future
// optimizations will clean it up.
- if (all_equal(PhiR->incoming_values())) {
+ auto NotPoison = [&](VPValue *V) {
+ // Don't remove poison from phis from the original loop.
+ return PhiR->getUnderlyingValue() || !isa<VPIRValue>(V) ||
+ !isa<PoisonValue>(cast<VPIRValue>(V)->getValue());
+ };
+
+ if (all_equal(make_filter_range(PhiR->incoming_values(), NotPoison))) {
PhiR->replaceAllUsesWith(PhiR->getIncomingValue(0));
PhiR->eraseFromParent();
continue;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index cae0633dbb185..dbdbf1027c0f7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -158,6 +158,14 @@ class LLVM_ABI_FOR_TEST VPValue {
VPValue *New,
llvm::function_ref<bool(VPUser &U, unsigned Idx)> ShouldReplace);
+ inline void
+ replaceUsesWithIf(VPValue *New,
+ llvm::function_ref<bool(VPUser *U)> ShouldReplace) {
+ return replaceUsesWithIf(New, [&ShouldReplace](VPUser &U, unsigned) {
+ return ShouldReplace(&U);
+ });
+ }
+
/// Returns the recipe defining this VPValue or nullptr if it is not defined
/// by a recipe, i.e. is a live-in.
VPRecipeBase *getDefiningRecipe();
>From b88529d493568dcba28f81c7b182b427648af326 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Sun, 25 Jan 2026 17:50:25 +0800
Subject: [PATCH 7/9] Remove unused VPTypeAnalysis
---
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 10bec7c91af1e..66fe4ba80ab37 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -1002,7 +1002,6 @@ void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
UR->getRegion() != Plan.getVectorLoopRegion();
};
- VPTypeAnalysis TypeInfo(Plan);
Builder.setInsertPoint(LatchSplit, LatchSplit->begin());
for (VPBlockBase *VPB : vp_depth_first_shallow(HeaderSplit)) {
auto *VPBB = cast<VPBasicBlock>(VPB);
>From df47d1e36bfef827997062dffd85ad3ff2f6cd77 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 29 Jan 2026 16:32:11 +0800
Subject: [PATCH 8/9] Assert no other recipes are in new latch
---
llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 66fe4ba80ab37..ca60a163d5ad4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -988,7 +988,9 @@ void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
assert(TermBranchOnCount &&
match(IVInc, m_Add(m_Specific(LoopRegion->getCanonicalIV()),
m_Specific(&Plan.getVFxUF()))) &&
- "Unexpected terminator");
+ std::next(IVInc->getDefiningRecipe()->getIterator()) ==
+ Latch->getTerminator()->getIterator() &&
+ "Unexpected canonical iv increment");
// Split the latch at the IV update, and branch to it from the header mask.
VPBasicBlock *LatchSplit =
>From 8c58ad6236b0f557ba5012a0fba4889bc528752e Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Thu, 29 Jan 2026 17:14:23 +0800
Subject: [PATCH 9/9] Iterate through defs instead of uses
---
.../Vectorize/VPlanConstruction.cpp | 27 ++++++++-----------
llvm/lib/Transforms/Vectorize/VPlanValue.h | 8 ------
2 files changed, 11 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index ca60a163d5ad4..968cfe15da6d4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -968,7 +968,7 @@ void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
- VPBasicBlock *HeaderSplit = Header->splitAt(Header->getFirstNonPhi());
+ Header->splitAt(Header->getFirstNonPhi());
// Create the header mask, insert it in the header and branch on it.
auto *IV =
@@ -997,26 +997,21 @@ void VPlanTransforms::foldTailByMasking(VPlan &Plan) {
Latch->splitAt(IVInc->getDefiningRecipe()->getIterator());
VPBlockUtils::connectBlocks(Header, LatchSplit);
- // Insert phis for any values used outside of the predicated body.
- auto NeedsPhi = [&Header, &Plan](VPUser *U) {
- auto *UR = cast<VPRecipeBase>(U);
- return UR->getParent() == Header ||
- UR->getRegion() != Plan.getVectorLoopRegion();
- };
-
+ // Insert phis for any values in the predicated body that are used outside.
Builder.setInsertPoint(LatchSplit, LatchSplit->begin());
- for (VPBlockBase *VPB : vp_depth_first_shallow(HeaderSplit)) {
- auto *VPBB = cast<VPBasicBlock>(VPB);
- if (VPBB == LatchSplit)
- continue;
+ for (VPBasicBlock *VPBB : {Header, Plan.getMiddleBlock()}) {
for (VPRecipeBase &R : *VPBB) {
- for (VPValue *V : R.definedValues()) {
- if (!any_of(V->users(), NeedsPhi))
+ for (VPValue *V : R.operands()) {
+ VPRecipeBase *VR = V->getDefiningRecipe();
+ if (!VR || !VR->getRegion() || VR->getParent() == LatchSplit ||
+ VR->getParent() == VPBB)
continue;
+ // TODO: For reduction phis, use phi value instead of poison.
VPValue *Poison = Plan.getOrAddLiveIn(
PoisonValue::get(V->getUnderlyingValue()->getType()));
- VPValue *Phi = Builder.createScalarPhi({V, Poison}, {});
- V->replaceUsesWithIf(Phi, NeedsPhi);
+ VPInstruction *Phi = Builder.createScalarPhi({V, Poison}, {});
+ V->replaceUsesWithIf(Phi,
+ [&Phi](VPUser &U, unsigned) { return &U != Phi; });
}
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index dbdbf1027c0f7..cae0633dbb185 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -158,14 +158,6 @@ class LLVM_ABI_FOR_TEST VPValue {
VPValue *New,
llvm::function_ref<bool(VPUser &U, unsigned Idx)> ShouldReplace);
- inline void
- replaceUsesWithIf(VPValue *New,
- llvm::function_ref<bool(VPUser *U)> ShouldReplace) {
- return replaceUsesWithIf(New, [&ShouldReplace](VPUser &U, unsigned) {
- return ShouldReplace(&U);
- });
- }
-
/// Returns the recipe defining this VPValue or nullptr if it is not defined
/// by a recipe, i.e. is a live-in.
VPRecipeBase *getDefiningRecipe();
More information about the llvm-commits
mailing list