[llvm] [VPlan][NFC] Extract addCurrentIterationPhi from addExplicitVectorLength (PR #182650)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Feb 20 19:50:04 PST 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Shih-Po Hung (arcbbb)
<details>
<summary>Changes</summary>
Refactor the creation of VPCurrentIterationPHIRecipe into a separate transformation pass `addCurrentIterationPhi`. This separates concerns and allows the CurrentIteration PHI to be created independently of EVL-based tail folding.
Changes:
- Add new `VPlanTransforms::addCurrentIterationPhi()` that creates VPCurrentIterationPHIRecipe and replaces uses of VPCanonicalIVPHIRecipe
- Add `vputils::getCurrentIterationPhi()` helper to locate the VPCurrentIterationPHIRecipe in a VPlan
- Add `vputils::getLoopIndex()` helper that returns either VPCurrentIterationPHIRecipe or VPCanonicalIVPHIRecipe
---
Full diff: https://github.com/llvm/llvm-project/pull/182650.diff
5 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+1)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp (+48-38)
- (modified) llvm/lib/Transforms/Vectorize/VPlanTransforms.h (+6-5)
- (modified) llvm/lib/Transforms/Vectorize/VPlanUtils.cpp (+38-6)
- (modified) llvm/lib/Transforms/Vectorize/VPlanUtils.h (+5)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 69d2b9f2c1a28..a9abacf38c4fa 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8145,6 +8145,7 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
RUN_VPLAN_PASS(VPlanTransforms::optimize, *Plan);
// TODO: try to put addExplicitVectorLength close to addActiveLaneMask
if (CM.foldTailWithEVL()) {
+ RUN_VPLAN_PASS(VPlanTransforms::addCurrentIterationPhi, *Plan);
RUN_VPLAN_PASS(VPlanTransforms::addExplicitVectorLength, *Plan,
CM.getMaxSafeElements());
RUN_VPLAN_PASS(VPlanTransforms::optimizeEVLMasks, *Plan);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b4c74c8776fc3..885aadac21316 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -3207,16 +3207,43 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
HeaderMask->replaceAllUsesWith(EVLMask);
}
+/// Add a VPCurrentIterationPHIRecipe to \p Plan and replaces all uses of
+/// VPCanonicalIVPHIRecipe with VPCurrentIterationPHIRecipe, except for the
+/// canonical IV increment. After this transfomration, VPCanonicalIVPHIRecipe
+/// is used only for loop iterations counting.
+void VPlanTransforms::addCurrentIterationPhi(VPlan &Plan) {
+ if (Plan.hasScalarVFOnly())
+ return;
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
+
+ auto *CanonicalIVPHI = LoopRegion->getCanonicalIV();
+ auto *CanonicalIVIncrement =
+ cast<VPInstruction>(CanonicalIVPHI->getBackedgeValue());
+ auto *CanIVTy = LoopRegion->getCanonicalIVType();
+ VPValue *StartV = CanonicalIVPHI->getStartValue();
+ // Create the CurrentIteration recipe in the vector loop.
+ auto *CurrentIteration =
+ new VPCurrentIterationPHIRecipe(StartV, DebugLoc::getUnknown());
+ CurrentIteration->insertAfter(CanonicalIVPHI);
+ VPBuilder Builder(CanonicalIVIncrement);
+ auto *NextIter = Builder.createAdd(
+ CanonicalIVIncrement->getOperand(1), CurrentIteration,
+ CanonicalIVIncrement->getDebugLoc(), "current.iteration.next",
+ {CanonicalIVIncrement->hasNoUnsignedWrap(),
+ CanonicalIVIncrement->hasNoSignedWrap()});
+ CurrentIteration->addOperand(NextIter);
+
+ // Replace all uses of VPCanonicalIVPHIRecipe by
+ // VPCurrentIterationPHIRecipe except for the canonical IV increment.
+ CanonicalIVPHI->replaceAllUsesWith(CurrentIteration);
+ CanonicalIVIncrement->setOperand(0, CanonicalIVPHI);
+}
+
/// Converts a tail folded vector loop region to step by
/// VPInstruction::ExplicitVectorLength elements instead of VF elements each
/// iteration.
///
-/// - Add a VPCurrentIterationPHIRecipe and related recipes to \p Plan and
-/// replaces all uses except the canonical IV increment of
-/// VPCanonicalIVPHIRecipe with a VPCurrentIterationPHIRecipe.
-/// VPCanonicalIVPHIRecipe is used only for loop iterations counting after
-/// this transformation.
-///
/// - The header mask is replaced with a header mask based on the EVL.
///
/// - Plans with FORs have a new phi added to keep track of the EVL of the
@@ -3272,10 +3299,8 @@ void VPlanTransforms::addExplicitVectorLength(
auto *CanIVTy = LoopRegion->getCanonicalIVType();
VPValue *StartV = CanonicalIVPHI->getStartValue();
- // Create the CurrentIteration recipe in the vector loop.
- auto *CurrentIteration =
- new VPCurrentIterationPHIRecipe(StartV, DebugLoc::getUnknown());
- CurrentIteration->insertAfter(CanonicalIVPHI);
+ auto *CurrentIteration = vputils::getCurrentIterationPhi(Plan);
+ assert(CurrentIteration && "must have CurrentIteration");
VPBuilder Builder(Header, Header->getFirstNonPhi());
// Create the AVL (application vector length), starting from TC -> 0 in steps
// of EVL.
@@ -3293,22 +3318,21 @@ void VPlanTransforms::addExplicitVectorLength(
auto *VPEVL = Builder.createNaryOp(VPInstruction::ExplicitVectorLength, AVL,
DebugLoc::getUnknown(), "evl");
- auto *CanonicalIVIncrement =
- cast<VPInstruction>(CanonicalIVPHI->getBackedgeValue());
- Builder.setInsertPoint(CanonicalIVIncrement);
+ auto *NextIter = cast<VPInstruction>(CurrentIteration->getBackedgeValue());
+ assert(NextIter->getOperand(0) == &Plan.getVFxUF() &&
+ "CurrentIteration increment should be VFxUF");
+ Builder.setInsertPoint(NextIter);
VPValue *OpVPEVL = VPEVL;
auto *I32Ty = Type::getInt32Ty(Plan.getContext());
- OpVPEVL = Builder.createScalarZExtOrTrunc(
- OpVPEVL, CanIVTy, I32Ty, CanonicalIVIncrement->getDebugLoc());
-
- auto *NextIter = Builder.createAdd(OpVPEVL, CurrentIteration,
- CanonicalIVIncrement->getDebugLoc(),
- "current.iteration.next",
- {CanonicalIVIncrement->hasNoUnsignedWrap(),
- CanonicalIVIncrement->hasNoSignedWrap()});
- CurrentIteration->addOperand(NextIter);
+ OpVPEVL = Builder.createScalarZExtOrTrunc(OpVPEVL, CanIVTy, I32Ty,
+ NextIter->getDebugLoc());
+
+ NextIter->setOperand(0, OpVPEVL);
+ auto *CanonicalIVIncrement =
+ cast<VPInstruction>(CanonicalIVPHI->getBackedgeValue());
+ Builder.setInsertPoint(CanonicalIVIncrement);
VPValue *NextAVL =
Builder.createSub(AVLPhi, OpVPEVL, DebugLoc::getCompilerGenerated(),
"avl.next", {/*NUW=*/true, /*NSW=*/false});
@@ -3317,27 +3341,13 @@ void VPlanTransforms::addExplicitVectorLength(
fixupVFUsersForEVL(Plan, *VPEVL);
removeDeadRecipes(Plan);
- // Replace all uses of VPCanonicalIVPHIRecipe by
- // VPCurrentIterationPHIRecipe except for the canonical IV increment.
- CanonicalIVPHI->replaceAllUsesWith(CurrentIteration);
- CanonicalIVIncrement->setOperand(0, CanonicalIVPHI);
// TODO: support unroll factor > 1.
Plan.setUF(1);
}
void VPlanTransforms::convertToVariableLengthStep(VPlan &Plan) {
- // Find the vector loop entry by locating VPCurrentIterationPHIRecipe.
- // There should be only one VPCurrentIteration in the entire plan.
- VPCurrentIterationPHIRecipe *CurrentIteration = nullptr;
-
- for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
- vp_depth_first_shallow(Plan.getEntry())))
- for (VPRecipeBase &R : VPBB->phis())
- if (auto *PhiR = dyn_cast<VPCurrentIterationPHIRecipe>(&R)) {
- assert(!CurrentIteration &&
- "Found multiple CurrentIteration. Only one expected");
- CurrentIteration = PhiR;
- }
+ VPCurrentIterationPHIRecipe *CurrentIteration =
+ vputils::getCurrentIterationPhi(Plan);
// Early return if it is not variable-length stepping.
if (!CurrentIteration)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 972a18ebded63..3ece149b8b9c6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -285,11 +285,12 @@ struct VPlanTransforms {
VPlan &Plan,
const std::function<bool(BasicBlock *)> &BlockNeedsPredication);
- /// Add a VPCurrentIterationPHIRecipe and related recipes to \p Plan and
- /// replaces all uses except the canonical IV increment of
- /// VPCanonicalIVPHIRecipe with a VPCurrentIterationPHIRecipe.
- /// VPCanonicalIVPHIRecipe is only used to control the loop after
- /// this transformation.
+ /// Add a VPCurrentIterationPHIRecipe to \p Plan and replaces all uses of
+ /// VPCanonicalIVPHIRecipe with VPCurrentIterationPHIRecipe, except for the
+ /// canonical IV increment.
+ static void addCurrentIterationPhi(VPlan &Plan);
+ /// Convert a tail-folded vector loop to use explicit vector length (EVL)
+ /// instead of VF elements each iteration.
static void
addExplicitVectorLength(VPlan &Plan,
const std::optional<unsigned> &MaxEVLSafeElements);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index f5318bb1c6515..093cd741171c7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -570,13 +570,12 @@ vputils::getRecipesForUncountableExit(VPlan &Plan,
VPSingleDefRecipe *vputils::findHeaderMask(VPlan &Plan) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
SmallVector<VPValue *> WideCanonicalIVs;
- auto *FoundWidenCanonicalIVUser = find_if(
- LoopRegion->getCanonicalIV()->users(), IsaPred<VPWidenCanonicalIVRecipe>);
- assert(count_if(LoopRegion->getCanonicalIV()->users(),
- IsaPred<VPWidenCanonicalIVRecipe>) <= 1 &&
+ VPHeaderPHIRecipe *LoopIndex = getLoopIndex(Plan);
+ auto *FoundWidenCanonicalIVUser =
+ find_if(LoopIndex->users(), IsaPred<VPWidenCanonicalIVRecipe>);
+ assert(count_if(LoopIndex->users(), IsaPred<VPWidenCanonicalIVRecipe>) <= 1 &&
"Must have at most one VPWideCanonicalIVRecipe");
- if (FoundWidenCanonicalIVUser !=
- LoopRegion->getCanonicalIV()->users().end()) {
+ if (FoundWidenCanonicalIVUser != LoopIndex->users().end()) {
auto *WideCanonicalIV =
cast<VPWidenCanonicalIVRecipe>(*FoundWidenCanonicalIVUser);
WideCanonicalIVs.push_back(WideCanonicalIV);
@@ -665,3 +664,36 @@ VPInstruction *vputils::findComputeReductionResult(VPReductionPHIRecipe *PhiR) {
return vputils::findUserOf<VPInstruction::ComputeReductionResult>(
cast<VPSingleDefRecipe>(SelR));
}
+
+VPCurrentIterationPHIRecipe *vputils::getCurrentIterationPhi(VPlan &Plan) {
+ VPCurrentIterationPHIRecipe *CurrentIteration = nullptr;
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ if (LoopRegion) {
+ VPCanonicalIVPHIRecipe *CanIV = LoopRegion->getCanonicalIV();
+ if (std::next(CanIV->getIterator()) != CanIV->getParent()->end())
+ CurrentIteration = dyn_cast_or_null<VPCurrentIterationPHIRecipe>(
+ std::next(CanIV->getIterator()));
+ return CurrentIteration;
+ }
+
+ // Find the vector loop entry by locating VPCurrentIterationPHIRecipe.
+ // There should be only one VPCurrentIteration in the entire plan.
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_shallow(Plan.getEntry())))
+ for (VPRecipeBase &R : VPBB->phis())
+ if (auto *PhiR = dyn_cast<VPCurrentIterationPHIRecipe>(&R)) {
+ assert(!CurrentIteration &&
+ "Found multiple CurrentIteration. Only one expected");
+ CurrentIteration = PhiR;
+ }
+
+ return CurrentIteration;
+}
+
+VPHeaderPHIRecipe *vputils::getLoopIndex(VPlan &Plan) {
+ VPCurrentIterationPHIRecipe *CurrentIteration =
+ vputils::getCurrentIterationPhi(Plan);
+ if (CurrentIteration)
+ return CurrentIteration;
+ return Plan.getVectorLoopRegion()->getCanonicalIV();
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index a5692699d9d76..f322b28f372f1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -151,6 +151,11 @@ VPInstruction *findComputeReductionResult(VPReductionPHIRecipe *PhiR);
/// the header-mask pattern manually.
VPSingleDefRecipe *findHeaderMask(VPlan &Plan);
+/// Return VPCurrentIterationPHIRecipe if present, otherwise nullptr.
+VPCurrentIterationPHIRecipe *getCurrentIterationPhi(VPlan &Plan);
+/// Return the loop index PHI: VPCurrentIterationPHIRecipe for variable-length
+/// stepping loops, or VPCanonicalIVPHIRecipe otherwise.
+VPHeaderPHIRecipe *getLoopIndex(VPlan &Plan);
} // namespace vputils
//===----------------------------------------------------------------------===//
``````````
</details>
https://github.com/llvm/llvm-project/pull/182650
More information about the llvm-commits
mailing list