[llvm] [VPlan] Introduce VPNumProcessedElementsPHIRecipe for variable-length stepping (PR #177114)
Shih-Po Hung via llvm-commits
llvm-commits at lists.llvm.org
Thu Jan 22 04:45:42 PST 2026
https://github.com/arcbbb updated https://github.com/llvm/llvm-project/pull/177114
>From 49f4470d58f5fea98ad2d2f84c25df411dc59193 Mon Sep 17 00:00:00 2001
From: ShihPo Hung <shihpo.hung at sifive.com>
Date: Tue, 20 Jan 2026 23:05:47 -0800
Subject: [PATCH] [VPlan] Introduce VPNumProcessedElementsPHIRecipe for
variable-length stepping
This is groundwork for #151300, which aims to support first-faulting
loads in non-tail-folded early-exit loops.
Per #175900, we need a variable-length stepping transform that can be
shared between EVL and non-EVL loops.
The idea is to have an EVL-independent counter and transform for
tracking the cumulative number of processed elements.
This patch renames the existing counter (VPEVLBasedIVPHIRecipe) and
transform (canonicalizeEVLLoops) to be EVL-independent:
- Rename VPEVLBasedIVPHIRecipe to VPNumProcessedElementsPHIRecipe to
reflect its general purpose of tracking processed element count.
- In addExplicitVectorLength, switch the loop from up-counting to
down-counting by transforming:
(branch-on-count CanonicalIVInc, VTC)
to:
(branch-on-count (sub VTC, CanonicalIVInc), 0)
- Rename canonicalizeEVLLoops to convertToVariableLengthStep and update
the exit condition from:
(branch-on-count (sub VectorTripCount, CanonicalIVInc), 0)
-> (branch-on-count (sub TripCount, (add Step, CumulativeIV)), 0)
-> (branch-on-count (sub AVL, Step), 0) if AVL is available.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 8 +-
llvm/lib/Transforms/Vectorize/VPlan.h | 27 +--
.../Transforms/Vectorize/VPlanAnalysis.cpp | 4 +-
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 7 +-
.../Transforms/Vectorize/VPlanTransforms.cpp | 198 +++++++++++-------
.../Transforms/Vectorize/VPlanTransforms.h | 20 +-
llvm/lib/Transforms/Vectorize/VPlanValue.h | 2 +-
.../Transforms/Vectorize/VPlanVerifier.cpp | 5 +-
.../RISCV/vplan-riscv-vector-reverse.ll | 6 +-
...an-vp-intrinsics-fixed-order-recurrence.ll | 7 +-
.../RISCV/vplan-vp-intrinsics-reduction.ll | 13 +-
.../RISCV/vplan-vp-intrinsics.ll | 7 +-
.../Transforms/Vectorize/VPlanTest.cpp | 2 +-
13 files changed, 177 insertions(+), 129 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0811628fd56b3..512393a44b8fd 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4126,10 +4126,10 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
case VPDef::VPReplicateSC:
case VPDef::VPInstructionSC:
case VPDef::VPCanonicalIVPHISC:
+ case VPDef::VPNumProcessedElementsPHISC:
case VPDef::VPVectorPointerSC:
case VPDef::VPVectorEndPointerSC:
case VPDef::VPExpandSCEVSC:
- case VPDef::VPEVLBasedIVPHISC:
case VPDef::VPPredInstPHISC:
case VPDef::VPBranchOnMaskSC:
continue;
@@ -4659,7 +4659,7 @@ LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
return 1;
if (any_of(Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
- IsaPred<VPEVLBasedIVPHIRecipe>)) {
+ IsaPred<VPNumProcessedElementsPHIRecipe>)) {
LLVM_DEBUG(dbgs() << "LV: Preference for VP intrinsics indicated. "
"Unroll factor forced to be 1.\n");
return 1;
@@ -7467,8 +7467,8 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
// Expand BranchOnTwoConds after dissolution, when latch has direct access to
// its successors.
VPlanTransforms::expandBranchOnTwoConds(BestVPlan);
- // Canonicalize EVL loops after regions are dissolved.
- VPlanTransforms::canonicalizeEVLLoops(BestVPlan);
+ VPlanTransforms::convertToVariableLengthStep(BestVPlan,
+ CM.foldTailByMasking());
VPlanTransforms::materializeBackedgeTakenCount(BestVPlan, VectorPH);
VPlanTransforms::materializeVectorTripCount(
BestVPlan, VectorPH, CM.foldTailByMasking(),
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 94a19beb75a8f..d881fdaf7f315 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -541,7 +541,6 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPRecipeValue {
static inline bool classof(const VPRecipeBase *R) {
switch (R->getVPDefID()) {
case VPRecipeBase::VPDerivedIVSC:
- case VPRecipeBase::VPEVLBasedIVPHISC:
case VPRecipeBase::VPExpandSCEVSC:
case VPRecipeBase::VPExpressionSC:
case VPRecipeBase::VPInstructionSC:
@@ -560,6 +559,7 @@ class VPSingleDefRecipe : public VPRecipeBase, public VPRecipeValue {
case VPRecipeBase::VPBlendSC:
case VPRecipeBase::VPPredInstPHISC:
case VPRecipeBase::VPCanonicalIVPHISC:
+ case VPRecipeBase::VPNumProcessedElementsPHISC:
case VPRecipeBase::VPActiveLaneMaskPHISC:
case VPRecipeBase::VPFirstOrderRecurrencePHISC:
case VPRecipeBase::VPWidenPHISC:
@@ -3678,28 +3678,31 @@ class VPActiveLaneMaskPHIRecipe : public VPHeaderPHIRecipe {
};
/// A recipe for generating the phi node for the current index of elements,
-/// adjusted in accordance with EVL value. It starts at the start value of the
-/// canonical induction and gets incremented by EVL in each iteration of the
-/// vector loop.
-class VPEVLBasedIVPHIRecipe : public VPHeaderPHIRecipe {
+/// It starts at the start value of the canonical induction and gets incremented
+/// by the number of elements processed in each iteration of the vector loop.
+class VPNumProcessedElementsPHIRecipe : public VPHeaderPHIRecipe {
public:
- VPEVLBasedIVPHIRecipe(VPValue *StartIV, DebugLoc DL)
- : VPHeaderPHIRecipe(VPDef::VPEVLBasedIVPHISC, nullptr, StartIV, DL) {}
+ VPNumProcessedElementsPHIRecipe(VPValue *StartIV, DebugLoc DL)
+ : VPHeaderPHIRecipe(VPDef::VPNumProcessedElementsPHISC, nullptr, StartIV,
+ DL) {}
- ~VPEVLBasedIVPHIRecipe() override = default;
+ ~VPNumProcessedElementsPHIRecipe() override = default;
- VPEVLBasedIVPHIRecipe *clone() override {
- llvm_unreachable("cloning not implemented yet");
+ VPNumProcessedElementsPHIRecipe *clone() override {
+ auto *R =
+ new VPNumProcessedElementsPHIRecipe(getStartValue(), getDebugLoc());
+ R->addOperand(getBackedgeValue());
+ return R;
}
- VP_CLASSOF_IMPL(VPDef::VPEVLBasedIVPHISC)
+ VP_CLASSOF_IMPL(VPDef::VPNumProcessedElementsPHISC)
void execute(VPTransformState &State) override {
llvm_unreachable("cannot execute this recipe, should be replaced by a "
"scalar phi recipe");
}
- /// Return the cost of this VPEVLBasedIVPHIRecipe.
+ /// Return the cost of this VPNumProcessedElementsPHIRecipe.
InstructionCost computeCost(ElementCount VF,
VPCostContext &Ctx) const override {
// For now, match the behavior of the legacy cost model.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 39e7a044dfff5..c7013107ebc65 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -270,7 +270,7 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
TypeSwitch<const VPRecipeBase *, Type *>(V->getDefiningRecipe())
.Case<VPActiveLaneMaskPHIRecipe, VPCanonicalIVPHIRecipe,
VPFirstOrderRecurrencePHIRecipe, VPReductionPHIRecipe,
- VPWidenPointerInductionRecipe, VPEVLBasedIVPHIRecipe>(
+ VPWidenPointerInductionRecipe, VPNumProcessedElementsPHIRecipe>(
[this](const auto *R) {
// Handle header phi recipes, except VPWidenIntOrFpInduction
// which needs special handling due it being possibly truncated.
@@ -543,7 +543,7 @@ SmallVector<VPRegisterUsage, 8> llvm::calculateRegisterUsageForPlan(
if (VFs[J].isScalar() ||
isa<VPCanonicalIVPHIRecipe, VPReplicateRecipe, VPDerivedIVRecipe,
- VPEVLBasedIVPHIRecipe, VPScalarIVStepsRecipe>(VPV) ||
+ VPNumProcessedElementsPHIRecipe, VPScalarIVStepsRecipe>(VPV) ||
(isa<VPInstruction>(VPV) && vputils::onlyScalarValuesUsed(VPV)) ||
(isa<VPReductionPHIRecipe>(VPV) &&
(cast<VPReductionPHIRecipe>(VPV))->isInLoop())) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 64890ac9e9b92..add72210c0887 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -74,6 +74,7 @@ bool VPRecipeBase::mayWriteToMemory() const {
case VPWidenIntrinsicSC:
return cast<VPWidenIntrinsicRecipe>(this)->mayWriteToMemory();
case VPCanonicalIVPHISC:
+ case VPNumProcessedElementsPHISC:
case VPBranchOnMaskSC:
case VPDerivedIVSC:
case VPFirstOrderRecurrencePHISC:
@@ -4587,9 +4588,9 @@ void VPActiveLaneMaskPHIRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
#endif
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-void VPEVLBasedIVPHIRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const {
- O << Indent << "EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ";
+void VPNumProcessedElementsPHIRecipe::printRecipe(
+ raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const {
+ O << Indent << "Num-Processed-Elements-PHI ";
printAsOperand(O, SlotTracker);
O << " = phi ";
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a39b171ab4cd6..f8e3fff2a9ccc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2052,7 +2052,7 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
if (all_of(Header->phis(), [](VPRecipeBase &Phi) {
if (auto *R = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi))
return R->isCanonical();
- return isa<VPCanonicalIVPHIRecipe, VPEVLBasedIVPHIRecipe,
+ return isa<VPCanonicalIVPHIRecipe, VPNumProcessedElementsPHIRecipe,
VPFirstOrderRecurrencePHIRecipe, VPPhi>(&Phi);
})) {
for (VPRecipeBase &HeaderR : make_early_inc_range(Header->phis())) {
@@ -3134,7 +3134,7 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
/// VPInstruction::ExplicitVectorLength elements instead of VF elements each
/// iteration.
///
-/// - Add a VPEVLBasedIVPHIRecipe and related recipes to \p Plan and
+/// - Add a VPNumProcessedElementsPHIRecipe and related recipes to \p Plan and
/// replaces all uses except the canonical IV increment of
/// VPCanonicalIVPHIRecipe with a VPEVLBasedIVPHIRecipe.
/// VPCanonicalIVPHIRecipe is used only for loop iterations counting after
@@ -3146,6 +3146,8 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
/// previous iteration, and VPFirstOrderRecurrencePHIRecipes are replaced with
/// @llvm.vp.splice.
///
+/// - Switches the loop from up-counting to down-counting.
+///
/// The function uses the following definitions:
/// %StartV is the canonical induction start value.
///
@@ -3156,13 +3158,15 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
///
/// vector.body:
/// ...
-/// %EVLPhi = EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI [ %StartV, %vector.ph ],
-/// [ %NextEVLIV, %vector.body ]
+/// %NumProcessedElementsPHI = Num-Processed-Elements-PHI [ %StartV, %vector.ph
+/// ],
+/// [ %NextIV,
+/// %vector.body ]
/// %AVL = phi [ trip-count, %vector.ph ], [ %NextAVL, %vector.body ]
/// %VPEVL = EXPLICIT-VECTOR-LENGTH %AVL
/// ...
/// %OpEVL = cast i32 %VPEVL to IVSize
-/// %NextEVLIV = add IVSize %OpEVL, %EVLPhi
+/// %NextIV = add IVSize %OpEVL, %NumProcessedElementsPHI
/// %NextAVL = sub IVSize nuw %AVL, %OpEVL
/// ...
///
@@ -3172,15 +3176,17 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
///
/// vector.body:
/// ...
-/// %EVLPhi = EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI [ %StartV, %vector.ph ],
-/// [ %NextEVLIV, %vector.body ]
+/// %NumProcessedElementsPHI = Num-Processed-Elements-PHI [ %StartV, %vector.ph
+/// ],
+/// [ %NextIV,
+/// %vector.body ]
/// %AVL = phi [ trip-count, %vector.ph ], [ %NextAVL, %vector.body ]
/// %cmp = cmp ult %AVL, MaxSafeElements
/// %SAFE_AVL = select %cmp, %AVL, MaxSafeElements
/// %VPEVL = EXPLICIT-VECTOR-LENGTH %SAFE_AVL
/// ...
/// %OpEVL = cast i32 %VPEVL to IVSize
-/// %NextEVLIV = add IVSize %OpEVL, %EVLPhi
+/// %NextIV = add IVSize %OpEVL, %NumProcessedElementsPHI
/// %NextAVL = sub IVSize nuw %AVL, %OpEVL
/// ...
///
@@ -3195,9 +3201,10 @@ void VPlanTransforms::addExplicitVectorLength(
auto *CanIVTy = LoopRegion->getCanonicalIVType();
VPValue *StartV = CanonicalIVPHI->getStartValue();
- // Create the ExplicitVectorLengthPhi recipe in the main loop.
- auto *EVLPhi = new VPEVLBasedIVPHIRecipe(StartV, DebugLoc::getUnknown());
- EVLPhi->insertAfter(CanonicalIVPHI);
+ // Create the NumProcessedElementsPHI recipe in the main loop.
+ auto *NumProcessedElementsPHI =
+ new VPNumProcessedElementsPHIRecipe(StartV, DebugLoc::getUnknown());
+ NumProcessedElementsPHI->insertAfter(CanonicalIVPHI);
VPBuilder Builder(Header, Header->getFirstNonPhi());
// Create the AVL (application vector length), starting from TC -> 0 in steps
// of EVL.
@@ -3224,12 +3231,12 @@ void VPlanTransforms::addExplicitVectorLength(
OpVPEVL = Builder.createScalarZExtOrTrunc(
OpVPEVL, CanIVTy, I32Ty, CanonicalIVIncrement->getDebugLoc());
- auto *NextEVLIV = Builder.createOverflowingOp(
- Instruction::Add, {OpVPEVL, EVLPhi},
+ auto *NumProcessedElementsNext = Builder.createOverflowingOp(
+ Instruction::Add, {OpVPEVL, NumProcessedElementsPHI},
{CanonicalIVIncrement->hasNoUnsignedWrap(),
CanonicalIVIncrement->hasNoSignedWrap()},
- CanonicalIVIncrement->getDebugLoc(), "index.evl.next");
- EVLPhi->addOperand(NextEVLIV);
+ CanonicalIVIncrement->getDebugLoc(), "cumulative.iv.next");
+ NumProcessedElementsPHI->addOperand(NumProcessedElementsNext);
VPValue *NextAVL = Builder.createOverflowingOp(
Instruction::Sub, {AVLPhi, OpVPEVL}, {/*hasNUW=*/true, /*hasNSW=*/false},
@@ -3240,89 +3247,122 @@ void VPlanTransforms::addExplicitVectorLength(
removeDeadRecipes(Plan);
// Replace all uses of VPCanonicalIVPHIRecipe by
- // VPEVLBasedIVPHIRecipe except for the canonical IV increment.
- CanonicalIVPHI->replaceAllUsesWith(EVLPhi);
+ // VPNumProcessedElementsPHIRecipe except for the canonical IV increment.
+ CanonicalIVPHI->replaceAllUsesWith(NumProcessedElementsPHI);
CanonicalIVIncrement->setOperand(0, CanonicalIVPHI);
+
// TODO: support unroll factor > 1.
Plan.setUF(1);
+
+ // Switch the loop from up-counting to down counting.
+ // convert (branch-on-count (CanonicalInc, VTC)
+ // -> (branch-on-count (sub VTC, CanonicalIVInc), 0)
+ VPBasicBlock *LatchVPBB = LoopRegion->getExitingBasicBlock();
+ auto *LatchExitingBranch = cast<VPInstruction>(LatchVPBB->getTerminator());
+ if (match(LatchExitingBranch, m_BranchOnCond(m_True())))
+ return;
+ assert(match(LatchExitingBranch,
+ m_BranchOnCount(m_Specific(CanonicalIVIncrement),
+ m_Specific(&Plan.getVectorTripCount()))) &&
+ "Unexpected terminator");
+ Builder.setInsertPoint(LatchExitingBranch);
+ VPValue *RemainElementCount = Builder.createOverflowingOp(
+ Instruction::Sub, {&Plan.getVectorTripCount(), CanonicalIVIncrement},
+ {/*hasNUW=*/true, /*hasNSW=*/false}, DebugLoc::getCompilerGenerated(),
+ "remain.element.count");
+ auto *Zero = Plan.getOrAddLiveIn(ConstantInt::get(CanIVTy, 0));
+ LatchExitingBranch->setOperand(0, RemainElementCount);
+ LatchExitingBranch->setOperand(1, Zero);
}
-void VPlanTransforms::canonicalizeEVLLoops(VPlan &Plan) {
- // Find EVL loop entries by locating VPEVLBasedIVPHIRecipe.
- // There should be only one EVL PHI in the entire plan.
- VPEVLBasedIVPHIRecipe *EVLPhi = nullptr;
+void VPlanTransforms::convertToVariableLengthStep(VPlan &Plan,
+ bool TailByMasking) {
+ VPNumProcessedElementsPHIRecipe *NumProcessedElementsPHI = nullptr;
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_shallow(Plan.getEntry())))
for (VPRecipeBase &R : VPBB->phis())
- if (auto *PhiR = dyn_cast<VPEVLBasedIVPHIRecipe>(&R)) {
- assert(!EVLPhi && "Found multiple EVL PHIs. Only one expected");
- EVLPhi = PhiR;
+ if (auto *PhiR = dyn_cast<VPNumProcessedElementsPHIRecipe>(&R)) {
+ assert(!NumProcessedElementsPHI &&
+ "Found multiple NumProcessedElementsPHI. Only one expected");
+ NumProcessedElementsPHI = PhiR;
}
-
- // Early return if no EVL PHI is found.
- if (!EVLPhi)
+ if (!NumProcessedElementsPHI)
return;
- VPBasicBlock *HeaderVPBB = EVLPhi->getParent();
- VPValue *EVLIncrement = EVLPhi->getBackedgeValue();
- VPValue *AVL;
- [[maybe_unused]] bool FoundAVL =
- match(EVLIncrement,
- m_c_Add(m_ZExtOrSelf(m_EVL(m_VPValue(AVL))), m_Specific(EVLPhi)));
- assert(FoundAVL && "Didn't find AVL?");
-
- // The AVL may be capped to a safe distance.
- VPValue *SafeAVL;
- if (match(AVL, m_Select(m_VPValue(), m_VPValue(SafeAVL), m_VPValue())))
- AVL = SafeAVL;
-
- VPValue *AVLNext;
- [[maybe_unused]] bool FoundAVLNext =
- match(AVL, m_VPInstruction<Instruction::PHI>(
- m_Specific(Plan.getTripCount()), m_VPValue(AVLNext)));
- assert(FoundAVLNext && "Didn't find AVL backedge?");
-
- // Convert EVLPhi to concrete recipe.
- auto *ScalarR =
- VPBuilder(EVLPhi).createScalarPhi({EVLPhi->getStartValue(), EVLIncrement},
- EVLPhi->getDebugLoc(), "evl.based.iv");
- EVLPhi->replaceAllUsesWith(ScalarR);
- EVLPhi->eraseFromParent();
-
- // Replace CanonicalIVInc with EVL-PHI increment.
+ auto *NumProcessedElementsNext =
+ cast<VPInstruction>(&NumProcessedElementsPHI->getBackedgeRecipe());
+ VPBasicBlock *HeaderVPBB = NumProcessedElementsPHI->getParent();
auto *CanonicalIV = cast<VPPhi>(&*HeaderVPBB->begin());
- VPValue *Backedge = CanonicalIV->getIncomingValue(1);
- assert(match(Backedge, m_c_Add(m_Specific(CanonicalIV),
- m_Specific(&Plan.getVFxUF()))) &&
+
+ VPValue *CanonicalIVInc = CanonicalIV->getIncomingValue(1);
+ assert(match(CanonicalIVInc, m_c_Add(m_Specific(CanonicalIV),
+ m_Specific(&Plan.getVFxUF()))) &&
"Unexpected canonical iv");
- Backedge->replaceAllUsesWith(EVLIncrement);
- // Remove unused phi and increment.
- VPRecipeBase *CanonicalIVIncrement = Backedge->getDefiningRecipe();
- CanonicalIVIncrement->eraseFromParent();
- CanonicalIV->eraseFromParent();
+ // TODO: Support non-tail-folded loops with variable stepping.
+ assert(TailByMasking &&
+ "Variable stepping is only supported in tail-folded loops");
- // Replace the use of VectorTripCount in the latch-exiting block.
- // Before: (branch-on-cond (icmp eq EVLIVInc, VectorTripCount))
- // After: (branch-on-cond icmp eq AVLNext, 0)
VPBasicBlock *LatchExiting =
HeaderVPBB->getPredecessors()[1]->getEntryBasicBlock();
auto *LatchExitingBr = cast<VPInstruction>(LatchExiting->getTerminator());
- if (match(LatchExitingBr, m_BranchOnCond(m_True())))
- return;
-
- assert(match(LatchExitingBr, m_BranchOnCond(m_SpecificCmp(
- CmpInst::ICMP_EQ, m_VPValue(EVLIncrement),
- m_Specific(&Plan.getVectorTripCount())))) &&
- "Expected BranchOnCond with ICmp comparing EVL increment with vector "
- "trip count");
-
- Type *AVLTy = VPTypeAnalysis(Plan).inferScalarType(AVLNext);
- VPBuilder Builder(LatchExitingBr);
- LatchExitingBr->setOperand(0,
- Builder.createICmp(CmpInst::ICMP_EQ, AVLNext,
- Plan.getConstantInt(AVLTy, 0)));
+ if (!match(LatchExitingBr, m_BranchOnCond(m_True()))) {
+ assert(LatchExitingBr->getOpcode() == VPInstruction::BranchOnCond &&
+ "Unexpected terminator");
+ VPValue *ExitCond = LatchExitingBr->getOperand(0);
+ // Convert the exit condition from
+ // (icmp eq (sub VectorTripCount, CanonicalIVInc), 0)
+ // to
+ // (icmp eq (sub TripCount, NumProcessedElementsNext), 0)
+ //
+ // TODO: Support up-counting loops: (icmp eq CanonicalIVInc,
+ // VectorTripCount)
+ VPValue *Remain;
+ [[maybe_unused]] bool IsStripMiningLoop =
+ match(ExitCond, m_SpecificCmp(CmpInst::ICMP_EQ, m_VPValue(Remain),
+ m_ZeroInt())) &&
+ match(Remain, m_VPInstruction<Instruction::Sub>(
+ m_Specific(&Plan.getVectorTripCount()),
+ m_Specific(CanonicalIVInc)));
+ assert(IsStripMiningLoop && "Unexpected exit condition for EVL loops");
+ auto *Sub = cast<VPInstruction>(Remain);
+ Sub->setOperand(0, Plan.getTripCount());
+ Sub->setOperand(1, NumProcessedElementsNext);
+
+ VPValue *VariableStep = NumProcessedElementsNext->getOperand(0);
+
+ // Try to simplify (sub TC, NumProcessedElementsNext) with recurrence
+ // {TC,-,Step} if possible.
+ VPValue *RemainPhi = nullptr;
+ for (VPRecipeBase &R : HeaderVPBB->phis()) {
+ auto *Phi = dyn_cast<VPPhi>(&R);
+ if (!Phi || Phi->getOperand(0) != Plan.getTripCount())
+ continue;
+ if (match(Phi->getOperand(1),
+ m_Binary<Instruction::Sub>(m_Specific(Phi),
+ m_Specific(VariableStep)))) {
+ RemainPhi = Phi->getOperand(1);
+ break;
+ }
+ }
+ if (RemainPhi)
+ ExitCond->getDefiningRecipe()->setOperand(0, RemainPhi);
+ }
+ // Lower NumProcessedElementsPHI abstract recipe to a concrete scalar phi.
+ auto *ScalarR =
+ VPBuilder(NumProcessedElementsPHI)
+ .createScalarPhi({NumProcessedElementsPHI->getStartValue(),
+ NumProcessedElementsNext},
+ NumProcessedElementsPHI->getDebugLoc(),
+ "cumulative.iv");
+ NumProcessedElementsPHI->replaceAllUsesWith(ScalarR);
+ NumProcessedElementsPHI->eraseFromParent();
+ CanonicalIV->replaceAllUsesWith(ScalarR);
+ CanonicalIVInc->replaceAllUsesWith(NumProcessedElementsNext);
+ // Remove the now-unused CanonicalIV phi and its increment.
+ CanonicalIV->eraseFromParent();
+ CanonicalIVInc->getDefiningRecipe()->eraseFromParent();
}
void VPlanTransforms::replaceSymbolicStrides(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index e0d09a099647a..07cae0b8ea93b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -262,7 +262,7 @@ struct VPlanTransforms {
VPlan &Plan,
const std::function<bool(BasicBlock *)> &BlockNeedsPredication);
- /// Add a VPEVLBasedIVPHIRecipe and related recipes to \p Plan and
+ /// Add a VPNumProcessedElementsPHIRecipe and related recipes to \p Plan and
/// replaces all uses except the canonical IV increment of
/// VPCanonicalIVPHIRecipe with a VPEVLBasedIVPHIRecipe.
/// VPCanonicalIVPHIRecipe is only used to control the loop after
@@ -308,18 +308,14 @@ struct VPlanTransforms {
/// BranchOnCond instructions. Should be called after dissolveLoopRegions.
static void expandBranchOnTwoConds(VPlan &Plan);
- /// Transform EVL loops to use variable-length stepping after region
+ /// Transform loops with variable-length stepping after region
/// dissolution.
- ///
- /// Once loop regions are replaced with explicit CFG, EVL loops can step with
- /// variable vector lengths instead of fixed lengths. This transformation:
- /// * Makes EVL-Phi concrete.
- // * Removes CanonicalIV and increment.
- /// * Replaces the exit condition from
- /// (branch-on-count CanonicalIVInc, VectorTripCount)
- /// to
- /// (branch-on-cond eq AVLNext, 0)
- static void canonicalizeEVLLoops(VPlan &Plan);
+ /// This replaces the exit condition from
+ /// (branch-on-cond (icmp eq (sub VectorTripCount, CanonicalIVInc), 0))
+ /// to
+ /// (branch-on-cond (icmp eq (sub TripCount, NumProcessedElementsNext), 0))
+ /// and removes CanonicalIV and its increment.
+ static void convertToVariableLengthStep(VPlan &Plan, bool TailByMasking);
/// Lower abstract recipes to concrete ones, that can be codegen'd.
static void convertToConcreteRecipes(VPlan &Plan);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index b24c40f0326a1..79bd2f88a2aca 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -418,8 +418,8 @@ class VPDef {
// START: SubclassID for recipes that inherit VPHeaderPHIRecipe.
// VPHeaderPHIRecipe need to be kept together.
VPCanonicalIVPHISC,
+ VPNumProcessedElementsPHISC,
VPActiveLaneMaskPHISC,
- VPEVLBasedIVPHISC,
VPFirstOrderRecurrencePHISC,
VPWidenIntOrFpInductionSC,
VPWidenPointerInductionSC,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 2d6809d6f344e..d4bb194108db4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -203,9 +203,10 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const {
errs() << "EVL used by unexpected VPInstruction\n";
return false;
}
- if (!VerifyLate && !isa<VPEVLBasedIVPHIRecipe>(*I->users().begin())) {
+ if (!VerifyLate &&
+ !isa<VPNumProcessedElementsPHIRecipe>(*I->users().begin())) {
errs() << "Result of VPInstruction::Add with EVL operand is "
- "not used by VPEVLBasedIVPHIRecipe\n";
+ "not used by VPNumProcessedElementsPHIRecipe\n";
return false;
}
return true;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-riscv-vector-reverse.ll
index 8b6ce538474c7..c41cd21691692 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-riscv-vector-reverse.ll
@@ -25,7 +25,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: <x1> vector loop: {
; CHECK-NEXT: vector.body:
; CHECK-NEXT: EMIT vp<[[INDUCTION:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[INDEX_NEXT:%.+]]>
-; CHECK-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%.+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; CHECK-NEXT: Num-Processed-Elements-PHI vp<[[EVL_PHI:%.+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
; CHECK-NEXT: EMIT-SCALAR vp<[[AVL:%.+]]> = phi [ vp<[[OTC]]>, vector.ph ], [ vp<[[AVL_NEXT:%.+]]>, vector.body ]
; CHECK-NEXT: EMIT-SCALAR vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
; CHECK-NEXT: vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[EVL_PHI]]> * ir<-1>
@@ -44,7 +44,9 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[EVL]]>, vp<[[EVL_PHI]]>
; CHECK-NEXT: EMIT vp<[[AVL_NEXT]]> = sub nuw vp<[[AVL]]>, vp<[[EVL]]>
; CHECK-NEXT: EMIT vp<[[INDEX_NEXT]]> = add vp<[[INDUCTION]]>, vp<[[VFxUF]]>
-; CHECK-NEXT: EMIT branch-on-count vp<[[INDEX_NEXT]]>, vp<[[VTC]]>
+; CHECK-NEXT: EMIT vp<[[REMAIN:%.+]]> = sub nuw vp<[[VTC]]>, vp<[[INDEX_NEXT]]>
+; CHECK-NEXT: EMIT branch-on-count vp<[[REMAIN]]>, ir<0>
+
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): middle.block
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll
index cc76ea04978a3..f092656463643 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll
@@ -6,7 +6,7 @@
define void @first_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
-; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+; IF-EVL-NOT: Num-Processed-Elements-PHI
;
; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
@@ -24,7 +24,7 @@ define void @first_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
; IF-EVL: <x1> vector loop: {
; IF-EVL-NEXT: vector.body:
; IF-EVL-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
-; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT: Num-Processed-Elements-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
; IF-EVL-NEXT: FIRST-ORDER-RECURRENCE-PHI ir<[[FOR_PHI:%.+]]> = phi ir<33>, ir<[[LD:%.+]]>
; IF-EVL-NEXT: EMIT-SCALAR vp<[[AVL:%.+]]> = phi [ ir<%TC>, vector.ph ], [ vp<[[AVL_NEXT:%.+]]>, vector.body ]
; IF-EVL-NEXT: EMIT-SCALAR vp<[[PREV_EVL:%.+]]> = phi [ vp<[[VF32]]>, vector.ph ], [ vp<[[EVL:%.+]]>, vector.body ]
@@ -42,7 +42,8 @@ define void @first_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
; IF-EVL-NEXT: EMIT vp<[[AVL_NEXT]]> = sub nuw vp<[[AVL]]>, vp<[[CAST]]>
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
-; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
+; IF-EVL-NEXT: EMIT vp<[[REMAIN:%.+]]> = sub nuw vp<[[VTC]]>, vp<[[IV_NEXT_EXIT]]>
+; IF-EVL-NEXT: EMIT branch-on-count vp<[[REMAIN]]>, ir<0>
; IF-EVL-NEXT: No successors
; IF-EVL-NEXT: }
; IF-EVL-NEXT: Successor(s): middle.block
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
index a3293366c17bc..037a2fef460ec 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll
@@ -21,7 +21,7 @@
define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
-; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+; IF-EVL-NOT: Num-Processed-Elements-PHI
; IF-EVL-OUTLOOP: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
; IF-EVL-OUTLOOP-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
@@ -38,7 +38,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-OUTLOOP-NEXT: <x1> vector loop: {
; IF-EVL-OUTLOOP-NEXT: vector.body:
; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
-; IF-EVL-OUTLOOP-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-OUTLOOP-NEXT: Num-Processed-Elements-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
; IF-EVL-OUTLOOP-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, vp<[[RDX_SELECT:%.+]]>
; IF-EVL-OUTLOOP-NEXT: EMIT-SCALAR vp<[[AVL:%.+]]> = phi [ ir<%n>, vector.ph ], [ vp<[[AVL_NEXT:%.+]]>, vector.body ]
; IF-EVL-OUTLOOP-NEXT: EMIT-SCALAR vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
@@ -52,7 +52,9 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[AVL_NEXT]]> = sub nuw vp<[[AVL]]>, vp<[[CAST]]>
; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
-; IF-EVL-OUTLOOP-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
+; IF-EVL-OUTLOOP-NEXT: EMIT vp<[[REMAIN:%.+]]> = sub nuw vp<[[VTC]]>, vp<[[IV_NEXT_EXIT]]>
+; IF-EVL-OUTLOOP-NEXT: EMIT branch-on-count vp<[[REMAIN]]>, ir<0>
+
; IF-EVL-OUTLOOP-NEXT: No successors
; IF-EVL-OUTLOOP-NEXT: }
; IF-EVL-OUTLOOP-NEXT: Successor(s): middle.block
@@ -78,7 +80,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-INLOOP-NEXT: <x1> vector loop: {
; IF-EVL-INLOOP-NEXT: vector.body:
; IF-EVL-INLOOP-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
-; IF-EVL-INLOOP-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-INLOOP-NEXT: Num-Processed-Elements-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
; IF-EVL-INLOOP-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
; IF-EVL-INLOOP-NEXT: EMIT-SCALAR vp<[[AVL:%.+]]> = phi [ ir<%n>, vector.ph ], [ vp<[[AVL_NEXT:%.+]]>, vector.body ]
; IF-EVL-INLOOP-NEXT: EMIT-SCALAR vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
@@ -91,7 +93,8 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
; IF-EVL-INLOOP-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
; IF-EVL-INLOOP-NEXT: EMIT vp<[[AVL_NEXT]]> = sub nuw vp<[[AVL]]>, vp<[[CAST]]>
; IF-EVL-INLOOP-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
-; IF-EVL-INLOOP-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
+; IF-EVL-INLOOP-NEXT: EMIT vp<[[REMAIN:%.+]]> = sub nuw vp<[[VTC]]>, vp<[[IV_NEXT_EXIT]]>
+; IF-EVL-INLOOP-NEXT: EMIT branch-on-count vp<[[REMAIN]]>, ir<0>
; IF-EVL-INLOOP-NEXT: No successors
; IF-EVL-INLOOP-NEXT: }
; IF-EVL-INLOOP-NEXT: Successor(s): middle.block
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
index b1e0a8462dbfe..b280e39e46038 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
@@ -10,7 +10,7 @@
define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
-; IF-EVL-NOT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI
+; IF-EVL-NOT: Num-Processed-Elements-PHI
;
; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
@@ -23,7 +23,7 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
; IF-EVL-NEXT: <x1> vector loop: {
; IF-EVL-NEXT: vector.body:
; IF-EVL-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
-; IF-EVL-NEXT: EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; IF-EVL-NEXT: Num-Processed-Elements-PHI vp<[[EVL_PHI:%[0-9]+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
; IF-EVL-NEXT: EMIT-SCALAR vp<[[AVL:%.+]]> = phi [ ir<%N>, vector.ph ], [ vp<[[AVL_NEXT:%.+]]>, vector.body ]
; IF-EVL-NEXT: EMIT-SCALAR vp<%evl> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<%evl>
@@ -41,7 +41,8 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
; IF-EVL-NEXT: EMIT vp<[[AVL_NEXT]]> = sub nuw vp<[[AVL]]>, vp<[[CAST]]>
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
-; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
+; IF-EVL-NEXT: EMIT vp<[[REMAIN:%.+]]> = sub nuw vp<[[VTC]]>, vp<[[IV_NEXT_EXIT]]>
+; IF-EVL-NEXT: EMIT branch-on-count vp<[[REMAIN]]>, ir<0>
; IF-EVL-NEXT: No successors
; IF-EVL-NEXT: }
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index b49c57a1332cf..53a0bb705b061 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1726,7 +1726,7 @@ TEST(VPDoubleValueDefTest, traverseUseLists) {
TEST_F(VPRecipeTest, CastToVPSingleDefRecipe) {
IntegerType *Int32 = IntegerType::get(C, 32);
VPValue *Start = getPlan().getOrAddLiveIn(ConstantInt::get(Int32, 0));
- VPEVLBasedIVPHIRecipe R(Start, {});
+ VPNumProcessedElementsPHIRecipe R(Start, {});
VPRecipeBase *B = &R;
EXPECT_TRUE(isa<VPSingleDefRecipe>(B));
// TODO: check other VPSingleDefRecipes.
More information about the llvm-commits
mailing list