[llvm] [VPlan] Introduce generic variable-length step support (PR #177114)
Shih-Po Hung via llvm-commits
llvm-commits at lists.llvm.org
Wed Feb 4 01:31:30 PST 2026
https://github.com/arcbbb updated https://github.com/llvm/llvm-project/pull/177114
>From c95a2e5edc856c761c6688a4a20b96b8b60477c9 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Wed, 4 Feb 2026 00:32:22 -0800
Subject: [PATCH 1/2] [VPlan] Introduce VPCurrentIterationRecipe for
variable-length stepping
This is groundwork for #151300, which aims to support first-faulting
loads in non-tail-folded early-exit loops.
Per #175900, we need a variable-length stepping transform that can
shared between EVL and non-EVL loops.
The idea is to have an EVL-independent counter and transform for
tracking the cumulative number of processed elements.
This patch renames the existing counter (VPEVLBasedIVPHIRecipe) and
transform (canonicalizeEVLLoops) to be EVL-independent:
- Rename VPEVLBasedIVPHIRecipe to VPCurrentIterationRecipe to
reflect its general purpose of tracking processed element count.
- Rename canonicalizeEVLLoops to convertToVariableLengthStep.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 8 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 25 +++---
.../Transforms/Vectorize/VPlanAnalysis.cpp | 4 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 7 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 83 ++++++++++---------
.../Transforms/Vectorize/VPlanTransforms.h | 10 +--
.../Transforms/Vectorize/VPlanVerifier.cpp | 7 +-
.../RISCV/vplan-riscv-vector-reverse.ll | 2 +-
...an-vp-intrinsics-fixed-order-recurrence.ll | 4 +-
.../RISCV/vplan-vp-intrinsics-reduction.ll | 6 +-
.../RISCV/vplan-vp-intrinsics.ll | 4 +-
.../Transforms/Vectorize/VPlanTest.cpp | 2 +-
12 files changed, 84 insertions(+), 78 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 51895d1be27d7..1c8ed6bb9e946 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4130,10 +4130,10 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
case VPRecipeBase::VPReplicateSC:
case VPRecipeBase::VPInstructionSC:
case VPRecipeBase::VPCanonicalIVPHISC:
+ case VPRecipeBase::VPCurrentIterationSC:
case VPRecipeBase::VPVectorPointerSC:
case VPRecipeBase::VPVectorEndPointerSC:
case VPRecipeBase::VPExpandSCEVSC:
- case VPRecipeBase::VPEVLBasedIVPHISC:
case VPRecipeBase::VPPredInstPHISC:
case VPRecipeBase::VPBranchOnMaskSC:
continue;
@@ -4663,7 +4663,7 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
return 1;
if (any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
- IsaPred<VPEVLBasedIVPHIRecipe>)) {
+ IsaPred<VPCurrentIterationRecipe>)) {
LLVM_DEBUG(dbgs() << "LV: Preference for VP intrinsics indicated. "
"Unroll factor forced to be 1.\n");
return 1;
@@ -7452,8 +7452,8 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
// Expand BranchOnTwoConds after dissolution, when latch has direct access to
// its successors.
VPlanTransforms::expandBranchOnTwoConds(BestVPlan);
- // Canonicalize EVL loops after regions are dissolved.
- VPlanTransforms::canonicalizeEVLLoops(BestVPlan);
+ // Convert loops with variable-length stepping after regions are dissolved.
+ VPlanTransforms::convertToVariableLengthStep(BestVPlan);
VPlanTransforms::materializeBackedgeTakenCount(BestVPlan, VectorPH);
VPlanTransforms::materializeVectorTripCount(
BestVPlan, VectorPH, CM.foldTailByMasking(),
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index a5f314ac188d8..2e8c6de855eb1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -435,8 +435,8 @@ class LLVM_ABI_FOR_TEST VPRecipeBase
// START: SubclassID for recipes that inherit VPHeaderPHIRecipe.
// VPHeaderPHIRecipe need to be kept together.
VPCanonicalIVPHISC,
+ VPCurrentIterationSC,
VPActiveLaneMaskPHISC,
- VPEVLBasedIVPHISC,
VPFirstOrderRecurrencePHISC,
VPWidenIntOrFpInductionSC,
VPWidenPointerInductionSC,
@@ -598,7 +598,6 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPRecipeValue {
static inline bool classof(const VPRecipeBase *R) {
switch (R->getVPRecipeID()) {
case VPRecipeBase::VPDerivedIVSC:
- case VPRecipeBase::VPEVLBasedIVPHISC:
case VPRecipeBase::VPExpandSCEVSC:
case VPRecipeBase::VPExpressionSC:
case VPRecipeBase::VPInstructionSC:
@@ -617,6 +616,7 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPRecipeValue {
case VPRecipeBase::VPBlendSC:
case VPRecipeBase::VPPredInstPHISC:
case VPRecipeBase::VPCanonicalIVPHISC:
+ case VPRecipeBase::VPCurrentIterationSC:
case VPRecipeBase::VPActiveLaneMaskPHISC:
case VPRecipeBase::VPFirstOrderRecurrencePHISC:
case VPRecipeBase::VPWidenPHISC:
@@ -3756,30 +3756,29 @@ class VPActiveLaneMaskPHIRecipe : public VPHeaderPHIRecipe {
#endif
};
-/// A recipe for generating the phi node for the current index of elements,
-/// adjusted in accordance with EVL value. It starts at the start value of the
-/// canonical induction and gets incremented by EVL in each iteration of the
-/// vector loop.
-class VPEVLBasedIVPHIRecipe : public VPHeaderPHIRecipe {
+/// A recipe for generating the phi node for the current index of elements.
+/// It starts at the start value of the canonical induction and gets incremented
+/// by the number of elements processed in each iteration of the vector loop.
+class VPCurrentIterationRecipe : public VPHeaderPHIRecipe {
public:
- VPEVLBasedIVPHIRecipe(VPValue *StartIV, DebugLoc DL)
- : VPHeaderPHIRecipe(VPRecipeBase::VPEVLBasedIVPHISC, nullptr, StartIV,
+ VPCurrentIterationRecipe(VPValue *StartIV, DebugLoc DL)
+ : VPHeaderPHIRecipe(VPRecipeBase::VPCurrentIterationSC, nullptr, StartIV,
DL) {}
- ~VPEVLBasedIVPHIRecipe() override = default;
+ ~VPCurrentIterationRecipe() override = default;
- VPEVLBasedIVPHIRecipe *clone() override {
+ VPCurrentIterationRecipe *clone() override {
llvm_unreachable("cloning not implemented yet");
}
- VP_CLASSOF_IMPL(VPRecipeBase::VPEVLBasedIVPHISC)
+ VP_CLASSOF_IMPL(VPRecipeBase::VPCurrentIterationSC)
void execute(VPTransformState &State) override {
llvm_unreachable("cannot execute this recipe, should be replaced by a "
"scalar phi recipe");
}
- /// Return the cost of this VPEVLBasedIVPHIRecipe.
+ /// Return the cost of this VPCurrentIterationRecipe.
InstructionCost computeCost(ElementCount VF,
VPCostContext &Ctx) const override {
// For now, match the behavior of the legacy cost model.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 8fbe7d93e6f45..b25e85aec4d29 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -269,7 +269,7 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
TypeSwitch<const VPRecipeBase *, Type *>(V->getDefiningRecipe())
.Case<VPActiveLaneMaskPHIRecipe, VPCanonicalIVPHIRecipe,
VPFirstOrderRecurrencePHIRecipe, VPReductionPHIRecipe,
- VPWidenPointerInductionRecipe, VPEVLBasedIVPHIRecipe>(
+ VPWidenPointerInductionRecipe, VPCurrentIterationRecipe>(
[this](const auto *R) {
// Handle header phi recipes, except VPWidenIntOrFpInduction
// which needs special handling due it being possibly truncated.
@@ -542,7 +542,7 @@ SmallVector<VPRegisterUsage, 8> llvm::calculateRegisterUsageForPlan(
if (VFs[J].isScalar() ||
isa<VPCanonicalIVPHIRecipe, VPReplicateRecipe, VPDerivedIVRecipe,
- VPEVLBasedIVPHIRecipe, VPScalarIVStepsRecipe>(VPV) ||
+ VPCurrentIterationRecipe, VPScalarIVStepsRecipe>(VPV) ||
(isa<VPInstruction>(VPV) && vputils::onlyScalarValuesUsed(VPV)) ||
(isa<VPReductionPHIRecipe>(VPV) &&
(cast<VPReductionPHIRecipe>(VPV))->isInLoop())) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index fa395e7d07531..fcc2f6f187514 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -75,6 +75,7 @@ bool VPRecipeBase::mayWriteToMemory() const {
return cast<VPWidenIntrinsicRecipe>(this)->mayWriteToMemory();
case VPActiveLaneMaskPHISC:
case VPCanonicalIVPHISC:
+ case VPCurrentIterationSC:
case VPBranchOnMaskSC:
case VPDerivedIVSC:
case VPFirstOrderRecurrencePHISC:
@@ -4606,9 +4607,9 @@ void VPActiveLaneMaskPHIRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
#endif
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-void VPEVLBasedIVPHIRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const {
- O << Indent << "EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ";
+void VPCurrentIterationRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
+ O << Indent << "Current-Iteration-PHI ";
printAsOperand(O, SlotTracker);
O << " = phi ";
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 9a89145fbf775..a0226573de52b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2060,7 +2060,7 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
if (all_of(Header->phis(), [](VPRecipeBase &Phi) {
if (auto *R = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi))
return R->isCanonical();
- return isa<VPCanonicalIVPHIRecipe, VPEVLBasedIVPHIRecipe,
+ return isa<VPCanonicalIVPHIRecipe, VPCurrentIterationRecipe,
VPFirstOrderRecurrencePHIRecipe, VPPhi>(&Phi);
})) {
for (VPRecipeBase &HeaderR : make_early_inc_range(Header->phis())) {
@@ -3203,7 +3203,7 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
/// VPInstruction::ExplicitVectorLength elements instead of VF elements each
/// iteration.
///
-/// - Add a VPEVLBasedIVPHIRecipe and related recipes to \p Plan and
+/// - Add a VPCurrentIterationRecipe and related recipes to \p Plan and
/// replaces all uses except the canonical IV increment of
/// VPCanonicalIVPHIRecipe with a VPEVLBasedIVPHIRecipe.
/// VPCanonicalIVPHIRecipe is used only for loop iterations counting after
@@ -3225,13 +3225,13 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
///
/// vector.body:
/// ...
-/// %EVLPhi = EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI [ %StartV, %vector.ph ],
-/// [ %NextEVLIV, %vector.body ]
+/// %CurrentIteration = Current-Iteration-PHI [ %StartV, %vector.ph ],
+/// [ %NextIV, %vector.body ]
/// %AVL = phi [ trip-count, %vector.ph ], [ %NextAVL, %vector.body ]
/// %VPEVL = EXPLICIT-VECTOR-LENGTH %AVL
/// ...
/// %OpEVL = cast i32 %VPEVL to IVSize
-/// %NextEVLIV = add IVSize %OpEVL, %EVLPhi
+/// %NextIV = add IVSize %OpEVL, %CurrentIteration
/// %NextAVL = sub IVSize nuw %AVL, %OpEVL
/// ...
///
@@ -3241,15 +3241,15 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
///
/// vector.body:
/// ...
-/// %EVLPhi = EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI [ %StartV, %vector.ph ],
-/// [ %NextEVLIV, %vector.body ]
+/// %CurrentIteration = Current-Iteration-PHI [ %StartV, %vector.ph ],
+/// [ %NextIV, %vector.body ]
/// %AVL = phi [ trip-count, %vector.ph ], [ %NextAVL, %vector.body ]
/// %cmp = cmp ult %AVL, MaxSafeElements
/// %SAFE_AVL = select %cmp, %AVL, MaxSafeElements
/// %VPEVL = EXPLICIT-VECTOR-LENGTH %SAFE_AVL
/// ...
/// %OpEVL = cast i32 %VPEVL to IVSize
-/// %NextEVLIV = add IVSize %OpEVL, %EVLPhi
+/// %NextIV = add IVSize %OpEVL, %CurrentIteration
/// %NextAVL = sub IVSize nuw %AVL, %OpEVL
/// ...
///
@@ -3264,9 +3264,10 @@ void VPlanTransforms::addExplicitVectorLength(
auto *CanIVTy = LoopRegion->getCanonicalIVType();
VPValue *StartV = CanonicalIVPHI->getStartValue();
- // Create the ExplicitVectorLengthPhi recipe in the main loop.
- auto *EVLPhi = new VPEVLBasedIVPHIRecipe(StartV, DebugLoc::getUnknown());
- EVLPhi->insertAfter(CanonicalIVPHI);
+ // Create the CurrentIteration recipe in the main loop.
+ auto *CurrentIteration =
+ new VPCurrentIterationRecipe(StartV, DebugLoc::getUnknown());
+ CurrentIteration->insertAfter(CanonicalIVPHI);
VPBuilder Builder(Header, Header->getFirstNonPhi());
// Create the AVL (application vector length), starting from TC -> 0 in steps
// of EVL.
@@ -3293,11 +3294,12 @@ void VPlanTransforms::addExplicitVectorLength(
OpVPEVL = Builder.createScalarZExtOrTrunc(
OpVPEVL, CanIVTy, I32Ty, CanonicalIVIncrement->getDebugLoc());
- auto *NextEVLIV = Builder.createAdd(
- OpVPEVL, EVLPhi, CanonicalIVIncrement->getDebugLoc(), "index.evl.next",
- {CanonicalIVIncrement->hasNoUnsignedWrap(),
- CanonicalIVIncrement->hasNoSignedWrap()});
- EVLPhi->addOperand(NextEVLIV);
+ auto *NumProcessedElementsNext =
+ Builder.createAdd(OpVPEVL, CurrentIteration,
+ CanonicalIVIncrement->getDebugLoc(), "index.evl.next",
+ {CanonicalIVIncrement->hasNoUnsignedWrap(),
+ CanonicalIVIncrement->hasNoSignedWrap()});
+ CurrentIteration->addOperand(NumProcessedElementsNext);
VPValue *NextAVL =
Builder.createSub(AVLPhi, OpVPEVL, DebugLoc::getCompilerGenerated(),
@@ -3308,47 +3310,50 @@ void VPlanTransforms::addExplicitVectorLength(
removeDeadRecipes(Plan);
// Replace all uses of VPCanonicalIVPHIRecipe by
- // VPEVLBasedIVPHIRecipe except for the canonical IV increment.
- CanonicalIVPHI->replaceAllUsesWith(EVLPhi);
+ // VPCurrentIterationRecipe except for the canonical IV increment.
+ CanonicalIVPHI->replaceAllUsesWith(CurrentIteration);
CanonicalIVIncrement->setOperand(0, CanonicalIVPHI);
// TODO: support unroll factor > 1.
Plan.setUF(1);
}
-void VPlanTransforms::canonicalizeEVLLoops(VPlan &Plan) {
- // Find EVL loop entries by locating VPEVLBasedIVPHIRecipe.
- // There should be only one EVL PHI in the entire plan.
- VPEVLBasedIVPHIRecipe *EVLPhi = nullptr;
+void VPlanTransforms::convertToVariableLengthStep(VPlan &Plan) {
+ // Find the vector loop entry by locating VPCurrentIterationRecipe.
+ // There should be only one VPCurrentIteration in the entire plan.
+ VPCurrentIterationRecipe *CurrentIteration = nullptr;
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_shallow(Plan.getEntry())))
for (VPRecipeBase &R : VPBB->phis())
- if (auto *PhiR = dyn_cast<VPEVLBasedIVPHIRecipe>(&R)) {
- assert(!EVLPhi && "Found multiple EVL PHIs. Only one expected");
- EVLPhi = PhiR;
+ if (auto *PhiR = dyn_cast<VPCurrentIterationRecipe>(&R)) {
+ assert(!CurrentIteration &&
+ "Found multiple CurrentIteration. Only one expected");
+ CurrentIteration = PhiR;
}
- // Early return if no EVL PHI is found.
- if (!EVLPhi)
+ // Early return if it is not variable-length stepping.
+ if (!CurrentIteration)
return;
- VPBasicBlock *HeaderVPBB = EVLPhi->getParent();
- VPValue *EVLIncrement = EVLPhi->getBackedgeValue();
+ VPBasicBlock *HeaderVPBB = CurrentIteration->getParent();
+ VPValue *CurrentIterationIncr = CurrentIteration->getBackedgeValue();
- // Convert EVLPhi to concrete recipe.
+ // Convert CurrentIteration to concrete recipe.
auto *ScalarR =
- VPBuilder(EVLPhi).createScalarPhi({EVLPhi->getStartValue(), EVLIncrement},
- EVLPhi->getDebugLoc(), "evl.based.iv");
- EVLPhi->replaceAllUsesWith(ScalarR);
- EVLPhi->eraseFromParent();
-
- // Replace CanonicalIVInc with EVL-PHI increment.
+ VPBuilder(CurrentIteration)
+ .createScalarPhi(
+ {CurrentIteration->getStartValue(), CurrentIterationIncr},
+ CurrentIteration->getDebugLoc(), "evl.based.iv");
+ CurrentIteration->replaceAllUsesWith(ScalarR);
+ CurrentIteration->eraseFromParent();
+
+ // Replace CanonicalIVInc with CurrentIteration increment.
auto *CanonicalIV = cast<VPPhi>(&*HeaderVPBB->begin());
VPValue *Backedge = CanonicalIV->getIncomingValue(1);
assert(match(Backedge, m_c_Add(m_Specific(CanonicalIV),
m_Specific(&Plan.getVFxUF()))) &&
"Unexpected canonical iv");
- Backedge->replaceAllUsesWith(EVLIncrement);
+ Backedge->replaceAllUsesWith(CurrentIterationIncr);
// Remove unused phi and increment.
VPRecipeBase *CanonicalIVIncrement = Backedge->getDefiningRecipe();
@@ -3366,8 +3371,8 @@ void VPlanTransforms::convertEVLExitCond(VPlan &Plan) {
if (std::next(CanIV->getIterator()) == CanIV->getParent()->end())
return;
// The EVL IV is always immediately after the canonical IV.
- auto *EVLPhi =
- dyn_cast_or_null<VPEVLBasedIVPHIRecipe>(std::next(CanIV->getIterator()));
+ auto *EVLPhi = dyn_cast_or_null<VPCurrentIterationRecipe>(
+ std::next(CanIV->getIterator()));
if (!EVLPhi)
return;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 49d15f15ece94..631e33ff7dab1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -274,7 +274,7 @@ struct VPlanTransforms {
VPlan &Plan,
const std::function<bool(BasicBlock *)> &BlockNeedsPredication);
- /// Add a VPEVLBasedIVPHIRecipe and related recipes to \p Plan and
+ /// Add a VPCurrentIterationRecipe and related recipes to \p Plan and
/// replaces all uses except the canonical IV increment of
/// VPCanonicalIVPHIRecipe with a VPEVLBasedIVPHIRecipe.
/// VPCanonicalIVPHIRecipe is only used to control the loop after
@@ -326,14 +326,14 @@ struct VPlanTransforms {
/// BranchOnCond instructions. Should be called after dissolveLoopRegions.
static void expandBranchOnTwoConds(VPlan &Plan);
- /// Transform EVL loops to use variable-length stepping after region
+ /// Transform loops with variable-length stepping after region
/// dissolution.
///
- /// Once loop regions are replaced with explicit CFG, EVL loops can step with
+ /// Once loop regions are replaced with explicit CFG, loops can step with
/// variable vector lengths instead of fixed lengths. This transformation:
- /// * Makes EVL-Phi concrete.
+ /// * Makes CurrentIteration-Phi concrete.
// * Removes CanonicalIV and increment.
- static void canonicalizeEVLLoops(VPlan &Plan);
+ static void convertToVariableLengthStep(VPlan &Plan);
/// Lower abstract recipes to concrete ones, that can be codegen'd.
static void convertToConcreteRecipes(VPlan &Plan);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 9098b9ce8562d..a30f44f758c1b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -103,7 +103,7 @@ bool VPlanVerifier::verifyPhiRecipes(const VPBasicBlock *VPBB) {
return false;
}
- if (isa<VPEVLBasedIVPHIRecipe>(RecipeI) &&
+ if (isa<VPCurrentIterationRecipe>(RecipeI) &&
!isa_and_nonnull<VPCanonicalIVPHIRecipe>(std::prev(RecipeI))) {
errs() << "EVL based IV is not immediately after canonical IV\n";
return false;
@@ -209,9 +209,10 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const {
errs() << "EVL used by unexpected VPInstruction\n";
return false;
}
- if (!VerifyLate && !isa<VPEVLBasedIVPHIRecipe>(*I->users().begin())) {
+ if (!VerifyLate &&
+ !isa<VPCurrentIterationRecipe>(*I->users().begin())) {
errs() << "Result of VPInstruction::Add with EVL operand is "
- "not used by VPEVLBasedIVPHIRecipe\n";
+ "not used by VPCurrentIterationRecipe\n";
return false;
}
return true;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-riscv-vector-reverse.ll
index 8b6ce538474c7..60eb4c8bb8db7 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-riscv-vector-reverse.ll
@@ -25,7 +25,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[INDUCTION:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[INDEX_NEXT:%.+]]>
-; CHECK-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%.+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; CHECK-NEXT: Current-Iteration-PHI vp<[[EVL_PHI:%.+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
; CHECK-NEXT: EMIT-SCALAR vp<[[AVL:%.+]]> = phi [ vp<[[OTC]]>, vector.ph ], [ vp<[[AVL_NEXT:%.+]]>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[EVL_PHI]]> * ir<-1>
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll
index cc76ea04978a3..dd60d21e4cab1 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll
@@ -6,7 +6,7 @@
define void @first_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
-; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+; IF-EVL-NOT: Current-Iteration-PHI
;
; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
@@ -24,7 +24,7 @@ define void @first_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
; IF-EVL: <x1> vector loop: {
; IF-EVL-NEXT: vector.body:
; IF-EVL-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
-; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT: Current-Iteration-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
; IF-EVL-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<[[FOR_PHI:%.+]]> = phi ir<33>, ir<[[LD:%.+]]>
; IF-EVL-NEXT: EMIT-SCALAR vp<[[AVL:%.+]]> = phi [ ir<%TC>, vector.ph ], [ vp<[[AVL_NEXT:%.+]]>, vector.body ]
; IF-EVL-NEXT: EMIT-SCALAR vp<[[PREV_EVL:%.+]]> = phi [ vp<[[VF32]]>, vector.ph ], [ vp<[[EVL:%.+]]>, vector.body ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
index a3293366c17bc..e275d024f9977 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
@@ -21,7 +21,7 @@
define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
-; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+; IF-EVL-NOT: Current-Iteration-PHI
; IF-EVL-OUTLOOP: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
; IF-EVL-OUTLOOP-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
@@ -38,7 +38,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-OUTLOOP-NEXT: <x1> vector loop: {
; IF-EVL-OUTLOOP-NEXT: vector.body:
; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
-; IF-EVL-OUTLOOP-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-OUTLOOP-NEXT: Current-Iteration-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
; IF-EVL-OUTLOOP-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, vp<[[RDX_SELECT:%.+]]>
; IF-EVL-OUTLOOP-NEXT: EMIT-SCALAR vp<[[AVL:%.+]]> = phi [ ir<%n>, vector.ph ], [ vp<[[AVL_NEXT:%.+]]>, vector.body ]
; IF-EVL-OUTLOOP-NEXT: EMIT-SCALAR vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
@@ -78,7 +78,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-INLOOP-NEXT: <x1> vector loop: {
; IF-EVL-INLOOP-NEXT: vector.body:
; IF-EVL-INLOOP-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
-; IF-EVL-INLOOP-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-INLOOP-NEXT: Current-Iteration-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
; IF-EVL-INLOOP-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
; IF-EVL-INLOOP-NEXT: EMIT-SCALAR vp<[[AVL:%.+]]> = phi [ ir<%n>, vector.ph ], [ vp<[[AVL_NEXT:%.+]]>, vector.body ]
; IF-EVL-INLOOP-NEXT: EMIT-SCALAR vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
index b1e0a8462dbfe..6c93564df0bb5 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
@@ -10,7 +10,7 @@
define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
-; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+; IF-EVL-NOT: Current-Iteration-PHI
;
; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
@@ -23,7 +23,7 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
; IF-EVL-NEXT: <x1> vector loop: {
; IF-EVL-NEXT: vector.body:
; IF-EVL-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
-; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT: Current-Iteration-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
; IF-EVL-NEXT: EMIT-SCALAR vp<[[AVL:%.+]]> = phi [ ir<%N>, vector.ph ], [ vp<[[AVL_NEXT:%.+]]>, vector.body ]
; IF-EVL-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<%evl>
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 52634abcfb670..3a300560d0fd4 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1739,7 +1739,7 @@ TEST(VPDoubleValueDefTest, traverseUseLists) {
TEST_F(VPRecipeTest, CastToVPSingleDefRecipe) {
IntegerType *Int32 = IntegerType::get(C, 32);
VPValue *Start = getPlan().getOrAddLiveIn(ConstantInt::get(Int32, 0));
- VPEVLBasedIVPHIRecipe R(Start, {});
+ VPCurrentIterationRecipe R(Start, {});
VPRecipeBase *B = &R;
EXPECT_TRUE(isa<VPSingleDefRecipe>(B));
// TODO: check other VPSingleDefRecipes.
>From cd43c0cc46086434ad24065202caeaf2e92b9d7c Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Wed, 4 Feb 2026 01:31:09 -0800
Subject: [PATCH 2/2] Address comment
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1c8ed6bb9e946..00292528bab98 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4664,7 +4664,7 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
if (any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
IsaPred<VPCurrentIterationRecipe>)) {
- LLVM_DEBUG(dbgs() << "LV: Preference for VP intrinsics indicated. "
+ LLVM_DEBUG(dbgs() << "LV: Loop requires variable-length step. "
"Unroll factor forced to be 1.\n");
return 1;
}
More information about the llvm-commits
mailing list