[llvm] [VPlan] Verify and handle FOR legality during header phi creation (NFC). (PR #191298)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 23 04:02:24 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/191298
>From d251dde8b1f25cdd14d97ffe2ef9aa88975af05b Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 2 Apr 2026 13:33:43 +0100
Subject: [PATCH 1/5] [VPlan] Verify and handle FOR legality during header phi
creation (NFC).
Move the logic to validate FOR users and introduce the split directly to
header phi creation. It makes sense to introduce the header phi and the
splice together.
It also means sinking only needs to be done once, instead for each
VPlan.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 28 +-
.../Vectorize/VPlanConstruction.cpp | 207 ++++++++++-
.../Transforms/Vectorize/VPlanTransforms.cpp | 343 ++++--------------
.../Transforms/Vectorize/VPlanTransforms.h | 27 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 14 +
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 3 +
...an-vp-intrinsics-fixed-order-recurrence.ll | 3 +-
.../VPlan/vplan-print-after-all.ll | 1 -
.../Transforms/LoopVectorize/induction.ll | 16 +-
.../LoopVectorize/interleaved-accesses.ll | 2 +-
10 files changed, 316 insertions(+), 328 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a7bbd94a0c776..bbd6cfb02d638 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7985,10 +7985,11 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE, &LVer);
// Create recipes for header phis.
- VPlanTransforms::createHeaderPhiRecipes(
- *VPlan0, PSE, *OrigLoop, Legal->getInductionVars(),
- Legal->getReductionVars(), Legal->getFixedOrderRecurrences(),
- CM.getInLoopReductions(), Hints.allowReordering());
+ if (!VPlanTransforms::createHeaderPhiRecipes(
+ *VPlan0, PSE, *OrigLoop, Legal->getInductionVars(),
+ Legal->getReductionVars(), Legal->getFixedOrderRecurrences(),
+ CM.getInLoopReductions(), Hints.allowReordering()))
+ return;
VPlanTransforms::simplifyRecipes(*VPlan0);
// If we're vectorizing a loop with an uncountable exit, make sure that the
@@ -8205,7 +8206,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// TODO: We can't call runPass on these transforms yet, due to verifier
// failures.
- VPlanTransforms::addExitUsersForFirstOrderRecurrences(*Plan, Range);
+ VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers(*Plan, Range);
// ---------------------------------------------------------------------------
// Transform initial VPlan: Apply previously taken decisions, in order, to
@@ -8268,12 +8269,6 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
RUN_VPLAN_PASS(VPlanTransforms::dropPoisonGeneratingRecipes, *Plan,
BlockNeedsPredication);
- // Sink users of fixed-order recurrence past the recipe defining the previous
- // value and introduce FirstOrderRecurrenceSplice VPInstructions.
- if (!RUN_VPLAN_PASS(VPlanTransforms::adjustFixedOrderRecurrences, *Plan,
- Builder))
- return nullptr;
-
if (useActiveLaneMask(Style)) {
// TODO: Move checks to VPlanTransforms::addActiveLaneMask once
// TailFoldingStyle is visible there.
@@ -8297,11 +8292,12 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
OrigLoop, *LI, Legal->getWidestInductionType(),
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE);
- VPlanTransforms::createHeaderPhiRecipes(
- *Plan, PSE, *OrigLoop, Legal->getInductionVars(),
- MapVector<PHINode *, RecurrenceDescriptor>(),
- SmallPtrSet<const PHINode *, 1>(), SmallPtrSet<PHINode *, 1>(),
- /*AllowReordering=*/false);
+ if (!VPlanTransforms::createHeaderPhiRecipes(
+ *Plan, PSE, *OrigLoop, Legal->getInductionVars(),
+ MapVector<PHINode *, RecurrenceDescriptor>(),
+ SmallPtrSet<const PHINode *, 1>(), SmallPtrSet<PHINode *, 1>(),
+ /*AllowReordering=*/false))
+ return nullptr;
[[maybe_unused]] bool CanHandleExits = VPlanTransforms::handleEarlyExits(
*Plan, UncountableExitStyle::NoUncountableExit, OrigLoop, PSE, *DT,
Legal->getAssumptionCache());
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index c9234b88fb084..b6fe5b52f9531 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -698,7 +698,163 @@ createWidenInductionRecipe(PHINode *Phi, VPPhi *PhiR, VPIRValue *Start,
return WideIV;
}
-void VPlanTransforms::createHeaderPhiRecipes(
+/// Try to sink users of \p FOR after \p Previous. \returns true if sinking
+/// succeeded or was not necessary.
+static bool
+sinkRecurrenceUsersAfterPrevious(VPFirstOrderRecurrencePHIRecipe *FOR,
+ VPRecipeBase *Previous,
+ VPDominatorTree &VPDT) {
+ // If Previous is a live-in (no defining recipe), it naturally dominates all
+ // recipes in the loop, so no sinking is needed.
+ if (!Previous)
+ return true;
+
+ // Collect recipes that need sinking.
+ SmallVector<VPRecipeBase *> WorkList;
+ SmallPtrSet<VPRecipeBase *, 8> Seen;
+ Seen.insert(Previous);
+ auto TryToPushSinkCandidate = [&](VPRecipeBase *SinkCandidate) {
+ // The previous value must not depend on the users of the recurrence phi.
+ // In that case, FOR is not a fixed order recurrence.
+ if (SinkCandidate == Previous)
+ return false;
+
+ if (isa<VPHeaderPHIRecipe>(SinkCandidate) ||
+ !Seen.insert(SinkCandidate).second ||
+ VPDT.properlyDominates(Previous, SinkCandidate))
+ return true;
+
+ if (vputils::cannotHoistOrSinkRecipe(*SinkCandidate))
+ return false;
+
+ WorkList.push_back(SinkCandidate);
+ return true;
+ };
+
+ // Recursively sink users of FOR after Previous.
+ WorkList.push_back(FOR);
+ for (unsigned I = 0; I != WorkList.size(); ++I) {
+ VPRecipeBase *Current = WorkList[I];
+ assert(Current->getNumDefinedValues() == 1 &&
+ "only recipes with a single defined value expected");
+
+ for (VPUser *User : Current->getVPSingleValue()->users()) {
+ if (!TryToPushSinkCandidate(cast<VPRecipeBase>(User)))
+ return false;
+ }
+ }
+
+ // Keep recipes to sink ordered by dominance so earlier instructions are
+ // processed first.
+ sort(WorkList, [&VPDT](const VPRecipeBase *A, const VPRecipeBase *B) {
+ return VPDT.properlyDominates(A, B);
+ });
+
+ for (VPRecipeBase *SinkCandidate : WorkList) {
+ if (SinkCandidate == FOR)
+ continue;
+
+ SinkCandidate->moveAfter(Previous);
+ Previous = SinkCandidate;
+ }
+ return true;
+}
+
+/// Try to hoist \p Previous and its operands before all users of \p FOR.
+/// \returns true if hoisting succeeded or was not necessary.
+static bool hoistPreviousBeforeFORUsers(VPFirstOrderRecurrencePHIRecipe *FOR,
+ VPRecipeBase *Previous,
+ VPDominatorTree &VPDT) {
+ if (vputils::cannotHoistOrSinkRecipe(*Previous))
+ return false;
+
+ // Collect recipes that need hoisting.
+ SmallVector<VPRecipeBase *> HoistCandidates;
+ SmallPtrSet<VPRecipeBase *, 8> Visited;
+ // Find the closest hoist point by looking at all users of FOR and selecting
+ // the recipe dominating all other users.
+ VPRecipeBase *HoistPoint = nullptr;
+ for (VPUser *U : FOR->users()) {
+ auto *R = cast<VPRecipeBase>(U);
+ if (!HoistPoint || VPDT.properlyDominates(R, HoistPoint))
+ HoistPoint = R;
+ }
+ assert(all_of(FOR->users(),
+ [&VPDT, HoistPoint](VPUser *U) {
+ auto *R = cast<VPRecipeBase>(U);
+ return HoistPoint == R ||
+ VPDT.properlyDominates(HoistPoint, R);
+ }) &&
+ "HoistPoint must dominate all users of FOR");
+
+ auto NeedsHoisting = [HoistPoint, &VPDT,
+ &Visited](VPValue *HoistCandidateV) -> VPRecipeBase * {
+ VPRecipeBase *HoistCandidate = HoistCandidateV->getDefiningRecipe();
+ if (!HoistCandidate)
+ return nullptr;
+ // Hoist candidate was already visited, no need to hoist.
+ if (!Visited.insert(HoistCandidate).second)
+ return nullptr;
+ // Candidate is in the plan entry (preheader) or a header phi, dominates
+ // FOR users without hoisting.
+ VPBasicBlock *ParentVPBB = HoistCandidate->getParent();
+ if (ParentVPBB->getPlan()->getEntry() == ParentVPBB ||
+ isa<VPHeaderPHIRecipe>(HoistCandidate))
+ return nullptr;
+ // If we reached a recipe that dominates HoistPoint, we don't need to
+ // hoist the recipe.
+ if (VPDT.properlyDominates(HoistCandidate, HoistPoint))
+ return nullptr;
+ return HoistCandidate;
+ };
+
+ if (!NeedsHoisting(Previous->getVPSingleValue()))
+ return true;
+
+ // Recursively try to hoist Previous and its operands before all users of
+ // FOR.
+ HoistCandidates.push_back(Previous);
+
+ for (unsigned I = 0; I != HoistCandidates.size(); ++I) {
+ VPRecipeBase *Current = HoistCandidates[I];
+ assert(Current->getNumDefinedValues() == 1 &&
+ "only recipes with a single defined value expected");
+ if (vputils::cannotHoistOrSinkRecipe(*Current))
+ return false;
+
+ for (VPValue *Op : Current->operands()) {
+ // If we reach FOR, it means the original Previous depends on some other
+ // recurrence that in turn depends on FOR. If that is the case, we would
+ // also need to hoist recipes involving the other FOR, which may break
+ // dependencies.
+ if (Op == FOR)
+ return false;
+
+ if (auto *R = NeedsHoisting(Op)) {
+ // Bail out if the recipe defines multiple values.
+ // TODO: Hoisting such recipes requires additional handling.
+ if (R->getNumDefinedValues() != 1)
+ return false;
+ HoistCandidates.push_back(R);
+ }
+ }
+ }
+
+ // Order recipes to hoist by dominance so earlier instructions are processed
+ // first.
+ sort(HoistCandidates, [&VPDT](const VPRecipeBase *A, const VPRecipeBase *B) {
+ return VPDT.properlyDominates(A, B);
+ });
+
+ for (VPRecipeBase *HoistCandidate : HoistCandidates) {
+ HoistCandidate->moveBefore(*HoistPoint->getParent(),
+ HoistPoint->getIterator());
+ }
+
+ return true;
+}
+
+bool VPlanTransforms::createHeaderPhiRecipes(
VPlan &Plan, PredicatedScalarEvolution &PSE, Loop &OrigLoop,
const MapVector<PHINode *, InductionDescriptor> &Inductions,
const MapVector<PHINode *, RecurrenceDescriptor> &Reductions,
@@ -707,8 +863,8 @@ void VPlanTransforms::createHeaderPhiRecipes(
// Retrieve the header manually from the intial plain-CFG VPlan.
VPBasicBlock *HeaderVPBB = cast<VPBasicBlock>(
Plan.getEntry()->getSuccessors()[1]->getSingleSuccessor());
- assert(VPDominatorTree(Plan).dominates(HeaderVPBB,
- HeaderVPBB->getPredecessors()[1]) &&
+ VPDominatorTree VPDT(Plan);
+ assert(VPDT.dominates(HeaderVPBB, HeaderVPBB->getPredecessors()[1]) &&
"header must dominate its latch");
auto CreateHeaderPhiRecipe = [&](VPPhi *PhiR) -> VPHeaderPHIRecipe * {
@@ -763,6 +919,50 @@ void VPlanTransforms::createHeaderPhiRecipes(
PhiR->eraseFromParent();
}
+ // Sink users of fixed-order recurrences past the recipe defining the
+ // previous value, introduce FirstOrderRecurrenceSplice VPInstructions, and
+ // replace FOR uses. At VPlan0, the VPlan mirrors the scalar IR, so the
+ // legality check (sink/hoist) is guaranteed to succeed by
+ // isFixedOrderRecurrence.
+ VPBuilder LoopBuilder;
+ for (VPRecipeBase &R : HeaderVPBB->phis()) {
+ auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R);
+ if (!FOR)
+ continue;
+
+ // Follow through FOR phi chains to find the actual Previous recipe.
+ // Fixed-order recurrences do not contain cycles, so this loop is
+ // guaranteed to terminate.
+ SmallPtrSet<VPFirstOrderRecurrencePHIRecipe *, 4> SeenPhis;
+ VPRecipeBase *Previous = FOR->getBackedgeValue()->getDefiningRecipe();
+ while (auto *PrevPhi =
+ dyn_cast_or_null<VPFirstOrderRecurrencePHIRecipe>(Previous)) {
+ assert(PrevPhi->getParent() == FOR->getParent());
+ assert(SeenPhis.insert(PrevPhi).second);
+ Previous = PrevPhi->getBackedgeValue()->getDefiningRecipe();
+ }
+
+ // Sink FOR users after Previous or hoist Previous before FOR users.
+ if (!sinkRecurrenceUsersAfterPrevious(FOR, Previous, VPDT) &&
+ !hoistPreviousBeforeFORUsers(FOR, Previous, VPDT))
+ return false;
+
+ // Create FirstOrderRecurrenceSplice and replace FOR uses.
+ VPBasicBlock *InsertBlock =
+ Previous ? Previous->getParent() : FOR->getParent();
+ auto InsertPt = (!Previous || isa<VPHeaderPHIRecipe>(Previous))
+ ? InsertBlock->getFirstNonPhi()
+ : std::next(Previous->getIterator());
+ LoopBuilder.setInsertPoint(InsertBlock, InsertPt);
+
+ auto *RecurSplice =
+ LoopBuilder.createNaryOp(VPInstruction::FirstOrderRecurrenceSplice,
+ {FOR, FOR->getBackedgeValue()});
+ FOR->replaceUsesWithIf(RecurSplice, [RecurSplice](VPUser &U, unsigned) {
+ return &U != RecurSplice;
+ });
+ }
+
for (const auto &[HeaderPhiR, ScalarPhiR] :
getMatchingPhisForScalarLoop(HeaderVPBB, Plan.getScalarPreheader())) {
auto *ResumePhiR = cast<VPPhi>(&ScalarPhiR);
@@ -776,6 +976,7 @@ void VPlanTransforms::createHeaderPhiRecipes(
? "bc.resume.val"
: "bc.merge.rdx");
}
+ return true;
}
void VPlanTransforms::createInLoopReductionRecipes(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b2e8b6a85a35a..8b3b906767eb9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -289,24 +289,6 @@ collectGroupedReplicateMemOps(
return Groups;
}
-/// Return true if we do not know how to (mechanically) hoist or sink \p R out
-/// of a loop region.
-static bool cannotHoistOrSinkRecipe(const VPRecipeBase &R) {
- // Assumes don't alias anything or throw; as long as they're guaranteed to
- // execute, they're safe to hoist.
- if (match(&R, m_Intrinsic<Intrinsic::assume>()))
- return false;
-
- // TODO: Relax checks in the future, e.g. we could also hoist reads, if their
- // memory location is not modified in the vector loop.
- if (R.mayHaveSideEffects() || R.mayReadFromMemory() || R.isPhi())
- return true;
-
- // Allocas cannot be hoisted.
- auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
- return RepR && RepR->getOpcode() == Instruction::Alloca;
-}
-
static bool sinkScalarOperands(VPlan &Plan) {
auto Iter = vp_depth_first_deep(Plan.getEntry());
bool ScalarVFOnly = Plan.hasScalarVFOnly();
@@ -325,7 +307,8 @@ static bool sinkScalarOperands(VPlan &Plan) {
if (!isa<VPReplicateRecipe, VPScalarIVStepsRecipe>(Candidate))
return;
- if (Candidate->getParent() == SinkTo || cannotHoistOrSinkRecipe(*Candidate))
+ if (Candidate->getParent() == SinkTo ||
+ vputils::cannotHoistOrSinkRecipe(*Candidate))
return;
if (auto *RepR = dyn_cast<VPReplicateRecipe>(Candidate))
@@ -2307,241 +2290,6 @@ void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
}
}
-/// Sink users of \p FOR after the recipe defining the previous value \p
-/// Previous of the recurrence. \returns true if all users of \p FOR could be
-/// re-arranged as needed or false if it is not possible.
-static bool
-sinkRecurrenceUsersAfterPrevious(VPFirstOrderRecurrencePHIRecipe *FOR,
- VPRecipeBase *Previous,
- VPDominatorTree &VPDT) {
- // If Previous is a live-in (no defining recipe), it naturally dominates all
- // recipes in the loop, so no sinking is needed.
- if (!Previous)
- return true;
-
- // Collect recipes that need sinking.
- SmallVector<VPRecipeBase *> WorkList;
- SmallPtrSet<VPRecipeBase *, 8> Seen;
- Seen.insert(Previous);
- auto TryToPushSinkCandidate = [&](VPRecipeBase *SinkCandidate) {
- // The previous value must not depend on the users of the recurrence phi. In
- // that case, FOR is not a fixed order recurrence.
- if (SinkCandidate == Previous)
- return false;
-
- if (isa<VPHeaderPHIRecipe>(SinkCandidate) ||
- !Seen.insert(SinkCandidate).second ||
- VPDT.properlyDominates(Previous, SinkCandidate))
- return true;
-
- if (cannotHoistOrSinkRecipe(*SinkCandidate))
- return false;
-
- WorkList.push_back(SinkCandidate);
- return true;
- };
-
- // Recursively sink users of FOR after Previous.
- WorkList.push_back(FOR);
- for (unsigned I = 0; I != WorkList.size(); ++I) {
- VPRecipeBase *Current = WorkList[I];
- assert(Current->getNumDefinedValues() == 1 &&
- "only recipes with a single defined value expected");
-
- for (VPUser *User : Current->getVPSingleValue()->users()) {
- if (!TryToPushSinkCandidate(cast<VPRecipeBase>(User)))
- return false;
- }
- }
-
- // Keep recipes to sink ordered by dominance so earlier instructions are
- // processed first.
- sort(WorkList, [&VPDT](const VPRecipeBase *A, const VPRecipeBase *B) {
- return VPDT.properlyDominates(A, B);
- });
-
- for (VPRecipeBase *SinkCandidate : WorkList) {
- if (SinkCandidate == FOR)
- continue;
-
- SinkCandidate->moveAfter(Previous);
- Previous = SinkCandidate;
- }
- return true;
-}
-
-/// Try to hoist \p Previous and its operands before all users of \p FOR.
-static bool hoistPreviousBeforeFORUsers(VPFirstOrderRecurrencePHIRecipe *FOR,
- VPRecipeBase *Previous,
- VPDominatorTree &VPDT) {
- if (cannotHoistOrSinkRecipe(*Previous))
- return false;
-
- // Collect recipes that need hoisting.
- SmallVector<VPRecipeBase *> HoistCandidates;
- SmallPtrSet<VPRecipeBase *, 8> Visited;
- VPRecipeBase *HoistPoint = nullptr;
- // Find the closest hoist point by looking at all users of FOR and selecting
- // the recipe dominating all other users.
- for (VPUser *U : FOR->users()) {
- auto *R = cast<VPRecipeBase>(U);
- if (!HoistPoint || VPDT.properlyDominates(R, HoistPoint))
- HoistPoint = R;
- }
- assert(all_of(FOR->users(),
- [&VPDT, HoistPoint](VPUser *U) {
- auto *R = cast<VPRecipeBase>(U);
- return HoistPoint == R ||
- VPDT.properlyDominates(HoistPoint, R);
- }) &&
- "HoistPoint must dominate all users of FOR");
-
- auto NeedsHoisting = [HoistPoint, &VPDT,
- &Visited](VPValue *HoistCandidateV) -> VPRecipeBase * {
- VPRecipeBase *HoistCandidate = HoistCandidateV->getDefiningRecipe();
- if (!HoistCandidate)
- return nullptr;
- VPRegionBlock *EnclosingLoopRegion =
- HoistCandidate->getParent()->getEnclosingLoopRegion();
- assert((!HoistCandidate->getRegion() ||
- HoistCandidate->getRegion() == EnclosingLoopRegion) &&
- "CFG in VPlan should still be flat, without replicate regions");
- // Hoist candidate was already visited, no need to hoist.
- if (!Visited.insert(HoistCandidate).second)
- return nullptr;
-
- // Candidate is outside loop region or a header phi, dominates FOR users w/o
- // hoisting.
- if (!EnclosingLoopRegion || isa<VPHeaderPHIRecipe>(HoistCandidate))
- return nullptr;
-
- // If we reached a recipe that dominates HoistPoint, we don't need to
- // hoist the recipe.
- if (VPDT.properlyDominates(HoistCandidate, HoistPoint))
- return nullptr;
- return HoistCandidate;
- };
-
- if (!NeedsHoisting(Previous->getVPSingleValue()))
- return true;
-
- // Recursively try to hoist Previous and its operands before all users of FOR.
- HoistCandidates.push_back(Previous);
-
- for (unsigned I = 0; I != HoistCandidates.size(); ++I) {
- VPRecipeBase *Current = HoistCandidates[I];
- assert(Current->getNumDefinedValues() == 1 &&
- "only recipes with a single defined value expected");
- if (cannotHoistOrSinkRecipe(*Current))
- return false;
-
- for (VPValue *Op : Current->operands()) {
- // If we reach FOR, it means the original Previous depends on some other
- // recurrence that in turn depends on FOR. If that is the case, we would
- // also need to hoist recipes involving the other FOR, which may break
- // dependencies.
- if (Op == FOR)
- return false;
-
- if (auto *R = NeedsHoisting(Op)) {
- // Bail out if the recipe defines multiple values.
- // TODO: Hoisting such recipes requires additional handling.
- if (R->getNumDefinedValues() != 1)
- return false;
- HoistCandidates.push_back(R);
- }
- }
- }
-
- // Order recipes to hoist by dominance so earlier instructions are processed
- // first.
- sort(HoistCandidates, [&VPDT](const VPRecipeBase *A, const VPRecipeBase *B) {
- return VPDT.properlyDominates(A, B);
- });
-
- for (VPRecipeBase *HoistCandidate : HoistCandidates) {
- HoistCandidate->moveBefore(*HoistPoint->getParent(),
- HoistPoint->getIterator());
- }
-
- return true;
-}
-
-bool VPlanTransforms::adjustFixedOrderRecurrences(VPlan &Plan,
- VPBuilder &LoopBuilder) {
- VPDominatorTree VPDT(Plan);
- VPTypeAnalysis TypeInfo(Plan);
-
- SmallVector<VPFirstOrderRecurrencePHIRecipe *> RecurrencePhis;
- for (VPRecipeBase &R :
- Plan.getVectorLoopRegion()->getEntry()->getEntryBasicBlock()->phis())
- if (auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R))
- RecurrencePhis.push_back(FOR);
-
- for (VPFirstOrderRecurrencePHIRecipe *FOR : RecurrencePhis) {
- SmallPtrSet<VPFirstOrderRecurrencePHIRecipe *, 4> SeenPhis;
- VPRecipeBase *Previous = FOR->getBackedgeValue()->getDefiningRecipe();
- // Fixed-order recurrences do not contain cycles, so this loop is guaranteed
- // to terminate.
- while (auto *PrevPhi =
- dyn_cast_or_null<VPFirstOrderRecurrencePHIRecipe>(Previous)) {
- assert(PrevPhi->getParent() == FOR->getParent());
- assert(SeenPhis.insert(PrevPhi).second);
- Previous = PrevPhi->getBackedgeValue()->getDefiningRecipe();
- }
-
- if (!sinkRecurrenceUsersAfterPrevious(FOR, Previous, VPDT) &&
- !hoistPreviousBeforeFORUsers(FOR, Previous, VPDT))
- return false;
-
- // Introduce a recipe to combine the incoming and previous values of a
- // fixed-order recurrence.
- VPBasicBlock *InsertBlock =
- Previous ? Previous->getParent() : FOR->getParent();
- if (!Previous || isa<VPHeaderPHIRecipe>(Previous))
- LoopBuilder.setInsertPoint(InsertBlock, InsertBlock->getFirstNonPhi());
- else
- LoopBuilder.setInsertPoint(InsertBlock,
- std::next(Previous->getIterator()));
-
- auto *RecurSplice =
- LoopBuilder.createNaryOp(VPInstruction::FirstOrderRecurrenceSplice,
- {FOR, FOR->getBackedgeValue()});
-
- FOR->replaceAllUsesWith(RecurSplice);
- // Set the first operand of RecurSplice to FOR again, after replacing
- // all users.
- RecurSplice->setOperand(0, FOR);
-
- // Check for users extracting at the penultimate active lane of the FOR.
- // If only a single lane is active in the current iteration, we need to
- // select the last element from the previous iteration (from the FOR phi
- // directly).
- for (VPUser *U : RecurSplice->users()) {
- if (!match(U, m_ExtractLane(m_LastActiveLane(m_VPValue()),
- m_Specific(RecurSplice))))
- continue;
-
- VPBuilder B(cast<VPInstruction>(U));
- VPValue *LastActiveLane = cast<VPInstruction>(U)->getOperand(0);
- Type *Ty = TypeInfo.inferScalarType(LastActiveLane);
- VPValue *Zero = Plan.getConstantInt(Ty, 0);
- VPValue *One = Plan.getConstantInt(Ty, 1);
- VPValue *PenultimateIndex = B.createSub(LastActiveLane, One);
- VPValue *PenultimateLastIter =
- B.createNaryOp(VPInstruction::ExtractLane,
- {PenultimateIndex, FOR->getBackedgeValue()});
- VPValue *LastPrevIter =
- B.createNaryOp(VPInstruction::ExtractLastLane, FOR);
-
- VPValue *Cmp = B.createICmp(CmpInst::ICMP_EQ, LastActiveLane, Zero);
- VPValue *Sel = B.createSelect(Cmp, LastPrevIter, PenultimateLastIter);
- cast<VPInstruction>(U)->replaceAllUsesWith(Sel);
- }
- }
- return true;
-}
-
void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) {
for (VPRecipeBase &R :
Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
@@ -2692,7 +2440,7 @@ static void licm(VPlan &Plan) {
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_shallow(LoopRegion->getEntry()))) {
for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- if (cannotHoistOrSinkRecipe(R))
+ if (vputils::cannotHoistOrSinkRecipe(R))
continue;
if (any_of(R.operands(), [](VPValue *Op) {
return !Op->isDefinedOutsideLoopRegions();
@@ -2712,7 +2460,7 @@ static void licm(VPlan &Plan) {
LoopRegion->getEntry());
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(POT)) {
for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
- if (cannotHoistOrSinkRecipe(R))
+ if (vputils::cannotHoistOrSinkRecipe(R))
continue;
if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
@@ -5620,11 +5368,12 @@ void VPlanTransforms::addBranchWeightToMiddleTerminator(
MiddleTerm->setMetadata(LLVMContext::MD_prof, BranchWeights);
}
-void VPlanTransforms::addExitUsersForFirstOrderRecurrences(VPlan &Plan,
- VFRange &Range) {
+void VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers(VPlan &Plan,
+ VFRange &Range) {
VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
auto *MiddleVPBB = Plan.getMiddleBlock();
VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
+ VPTypeAnalysis TypeInfo(Plan);
auto IsScalableOne = [](ElementCount VF) -> bool {
return VF == ElementCount::getScalable(1);
@@ -5635,11 +5384,31 @@ void VPlanTransforms::addExitUsersForFirstOrderRecurrences(VPlan &Plan,
if (!FOR)
continue;
+ // For VF vscale x 1, if vscale = 1, we are unable to extract the
+ // penultimate value of the recurrence. Instead we rely on the existing
+ // extract of the last element from the result of
+ // VPInstruction::FirstOrderRecurrenceSplice.
+ // TODO: Consider vscale_range info and UF.
+ if (LoopVectorizationPlanner::getDecisionAndClampRange(IsScalableOne,
+ Range))
+ return;
+
assert(VectorRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
"Cannot handle loops with uncountable early exits");
+ // Find the existing splice for this FOR, created in
+ // createHeaderPhiRecipes. All uses of FOR have already been replaced with
+ // RecurSplice there; only RecurSplice itself still references FOR.
+ auto It = find_if(FOR->users(), [&](VPUser *U) {
+ return match(U,
+ m_VPInstruction<VPInstruction::FirstOrderRecurrenceSplice>(
+ m_Specific(FOR), m_VPValue()));
+ });
+ assert(It != FOR->users().end() && "expected FirstOrderRecurrenceSplice");
+ auto *RecurSplice = cast<VPInstruction>(*It);
+
// This is the second phase of vectorizing first-order recurrences, creating
- // extract for users outside the loop. An overview of the transformation is
+ // extracts for users outside the loop. An overview of the transformation is
// described below. Suppose we have the following loop with some use after
// the loop of the last a[i-1],
//
@@ -5679,9 +5448,8 @@ void VPlanTransforms::addExitUsersForFirstOrderRecurrences(VPlan &Plan,
// i = phi [0, vector.ph], [i+4, vector.body]
// v1 = phi [v_init, vector.ph], [v2, vector.body]
// v2 = a[i, i+1, i+2, i+3]
- // b[i] = v2 - v1
- // // Next, third phase will introduce v1' = splice(v1(3), v2(0, 1, 2))
- // b[i, i+1, i+2, i+3] = v2 - v1
+ // v1' = splice(v1(3), v2(0, 1, 2))
+ // b[i, i+1, i+2, i+3] = v2 - v1'
// br cond, vector.body, middle.block
//
// middle.block:
@@ -5705,30 +5473,43 @@ void VPlanTransforms::addExitUsersForFirstOrderRecurrences(VPlan &Plan,
// lo = lcssa.phi [s1, scalar.body],
// [vector.recur.extract.for.phi, middle.block]
//
- // Now update VPIRInstructions modeling LCSSA phis in the exit block.
- // Extract the penultimate value of the recurrence and use it as operand for
- // the VPIRInstruction modeling the phi.
+ // Update extracts of the splice in the middle block: they extract the
+ // penultimate element of the recurrence.
for (VPRecipeBase &R : make_early_inc_range(
make_range(MiddleVPBB->getFirstNonPhi(), MiddleVPBB->end()))) {
- if (!match(&R, m_ExtractLastLaneOfLastPart(m_Specific(FOR))))
+ // For tail-folded loops, the extract may have been converted to
+ // ExtractLane(LastActiveLane, RecurSplice). If only a single lane is
+ // active, select the last element from the previous iteration instead.
+ if (match(&R, m_ExtractLane(m_LastActiveLane(m_VPValue()),
+ m_Specific(RecurSplice)))) {
+ auto *ExtractR = cast<VPInstruction>(&R);
+ VPBuilder B(ExtractR);
+ VPValue *LastActiveLane = ExtractR->getOperand(0);
+ Type *Ty = TypeInfo.inferScalarType(LastActiveLane);
+ VPValue *Zero = Plan.getConstantInt(Ty, 0);
+ VPValue *One = Plan.getConstantInt(Ty, 1);
+ VPValue *PenultimateIndex = B.createSub(LastActiveLane, One);
+ VPValue *PenultimateLastIter =
+ B.createNaryOp(VPInstruction::ExtractLane,
+ {PenultimateIndex, RecurSplice->getOperand(1)});
+ VPValue *LastPrevIter = B.createNaryOp(VPInstruction::ExtractLastLane,
+ RecurSplice->getOperand(0));
+
+ VPValue *Cmp = B.createICmp(CmpInst::ICMP_EQ, LastActiveLane, Zero);
+ VPValue *Sel = B.createSelect(Cmp, LastPrevIter, PenultimateLastIter);
+ ExtractR->replaceAllUsesWith(Sel);
+ }
+
+ if (!match(&R, m_ExtractLastLaneOfLastPart(m_Specific(RecurSplice))))
continue;
- // For VF vscale x 1, if vscale = 1, we are unable to extract the
- // penultimate value of the recurrence. Instead we rely on the existing
- // extract of the last element from the result of
- // VPInstruction::FirstOrderRecurrenceSplice.
- // TODO: Consider vscale_range info and UF.
- if (LoopVectorizationPlanner::getDecisionAndClampRange(IsScalableOne,
- Range))
- return;
+ auto *ExtractR = cast<VPInstruction>(&R);
VPValue *PenultimateElement = MiddleBuilder.createNaryOp(
- VPInstruction::ExtractPenultimateElement, FOR->getBackedgeValue(), {},
- "vector.recur.extract.for.phi");
- for (VPUser *U : to_vector(cast<VPInstruction>(&R)->users())) {
- auto *ExitPhi = dyn_cast<VPIRPhi>(U);
- if (!ExitPhi)
- continue;
- ExitPhi->replaceUsesOfWith(cast<VPInstruction>(&R), PenultimateElement);
+ VPInstruction::ExtractPenultimateElement, RecurSplice->getOperand(1),
+ {}, "vector.recur.extract.for.phi");
+ for (VPUser *ExitU : to_vector(ExtractR->users())) {
+ if (auto *ExitPhi = dyn_cast<VPIRPhi>(ExitU))
+ ExitPhi->replaceUsesOfWith(ExtractR, PenultimateElement);
}
}
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 6312b823e5d33..414c30fa4324b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -134,8 +134,11 @@ struct VPlanTransforms {
/// Replace VPPhi recipes in \p Plan's header with corresponding
/// VPHeaderPHIRecipe subclasses for inductions, reductions, and
/// fixed-order recurrences. This processes all header phis and creates
- /// the appropriate widened recipe for each one.
- static void createHeaderPhiRecipes(
+ /// the appropriate widened recipe for each one. For fixed-order
+ /// recurrences, also creates FirstOrderRecurrenceSplice instructions and
+ /// sinks/hoists users as needed. Returns false if any fixed-order
+ /// recurrence cannot be handled.
+ static bool createHeaderPhiRecipes(
VPlan &Plan, PredicatedScalarEvolution &PSE, Loop &OrigLoop,
const MapVector<PHINode *, InductionDescriptor> &Inductions,
const MapVector<PHINode *, RecurrenceDescriptor> &Reductions,
@@ -212,16 +215,6 @@ struct VPlanTransforms {
OptimizationRemarkEmitter *ORE,
Loop *TheLoop);
- /// Try to have all users of fixed-order recurrences appear after the recipe
- /// defining their previous value, by either sinking users or hoisting recipes
- /// defining their previous value (and its operands). Then introduce
- /// FirstOrderRecurrenceSplice VPInstructions to combine the value from the
- /// recurrence phis and previous values.
- /// \returns true if all users of fixed-order recurrences could be re-arranged
- /// as needed or false if it is not possible. In the latter case, \p Plan is
- /// not valid.
- static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder);
-
/// Check if \p Plan contains any FMaxNum or FMinNum reductions. If they do,
/// try to update the vector loop to exit early if any input is NaN and resume
/// executing in the scalar loop to handle the NaNs there. Return false if
@@ -516,11 +509,11 @@ struct VPlanTransforms {
addBranchWeightToMiddleTerminator(VPlan &Plan, ElementCount VF,
std::optional<unsigned> VScaleForTuning);
- /// Handle users in the exit block for first order reductions in the original
- /// exit block. The penultimate value of recurrences is fed to their LCSSA phi
- /// users in the original exit block using the VPIRInstruction wrapping to the
- /// LCSSA phi.
- static void addExitUsersForFirstOrderRecurrences(VPlan &Plan, VFRange &Range);
+ /// Adjust first-order recurrence users in the middle block: create
+ /// penultimate element extracts for LCSSA phi users, and handle penultimate
+ /// extracts of the last active lane edge.
+ static void adjustFirstOrderRecurrenceMiddleUsers(VPlan &Plan,
+ VFRange &Range);
/// Optimize FindLast reductions selecting IVs (or expressions of IVs) by
/// converting them to FindIV reductions, if their IV range excludes a
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index e33a936dd9978..512036444b534 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -470,6 +470,20 @@ unsigned vputils::getVFScaleFactor(VPRecipeBase *R) {
return 1;
}
+bool vputils::cannotHoistOrSinkRecipe(const VPRecipeBase &R) {
+ // Assumes don't alias anything or throw; as long as they're guaranteed to
+ // execute, they're safe to hoist.
+ if (match(&R, m_Intrinsic<Intrinsic::assume>()))
+ return false;
+ // TODO: Relax checks in the future, e.g. we could also hoist reads, if their
+ // memory location is not modified in the vector loop.
+ if (R.mayHaveSideEffects() || R.mayReadFromMemory() || R.isPhi())
+ return true;
+ // Allocas cannot be hoisted.
+ auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
+ return RepR && RepR->getOpcode() == Instruction::Alloca;
+}
+
std::optional<VPValue *>
vputils::getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
SmallVectorImpl<VPInstruction *> &GEPs,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 2cab5967b42f7..4985f0c330dab 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -72,6 +72,9 @@ VPBasicBlock *getFirstLoopHeader(VPlan &Plan, VPDominatorTree &VPDT);
/// one.
unsigned getVFScaleFactor(VPRecipeBase *R);
+/// Return true if we do not know how to (mechanically) hoist or sink \p R.
+bool cannotHoistOrSinkRecipe(const VPRecipeBase &R);
+
/// Returns the VPValue representing the uncountable exit comparison used by
/// AnyOf if the recipes it depends on can be traced back to live-ins and
/// the addresses (in GEP/PtrAdd form) of any (non-masked) load used in
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll
index f87b3326808fd..389264d6579d1 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll
@@ -5,8 +5,9 @@
define void @first_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
; IF-EVL-NOT: CURRENT-ITERATION-PHI
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1},UF={1}'
;
-; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 2,vscale x 4},UF={1}' {
; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
index a698636aa2349..586871adab4df 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
@@ -16,7 +16,6 @@
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createInterleaveGroups
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::replaceSymbolicStrides
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::dropPoisonGeneratingRecipes
-; CHECK: VPlan for loop in 'foo' after VPlanTransforms::adjustFixedOrderRecurrences
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::truncateToMinimalBitwidths
; CHECK: VPlan for loop in 'foo' after removeRedundantCanonicalIVs
; CHECK: VPlan for loop in 'foo' after removeRedundantInductionCasts
diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll
index ff72e0d428d64..0d90d77f406c7 100644
--- a/llvm/test/Transforms/LoopVectorize/induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction.ll
@@ -5666,10 +5666,10 @@ define void @pr52460_first_order_recurrence_truncated_iv(ptr noalias %src, ptr %
; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i32> [ <i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[VEC_IND:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[INDEX]] to i32
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND]], <2 x i32> <i32 1, i32 2>
; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[SRC:%.*]], align 4
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP3]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND]], <2 x i32> <i32 1, i32 2>
; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <2 x i32> [[BROADCAST_SPLAT]], [[TMP2]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[DST:%.*]], i32 [[TMP0]]
; CHECK-NEXT: [[TMP6:%.*]] = add <2 x i32> [[VEC_IND]], [[TMP4]]
@@ -5692,10 +5692,10 @@ define void @pr52460_first_order_recurrence_truncated_iv(ptr noalias %src, ptr %
; IND-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; IND-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i32> [ <i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[VEC_IND:%.*]], [[VECTOR_BODY]] ]
; IND-NEXT: [[VEC_IND]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; IND-NEXT: [[TMP0:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND]], <2 x i32> <i32 1, i32 2>
; IND-NEXT: [[TMP1:%.*]] = load i32, ptr [[SRC:%.*]], align 4
; IND-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP1]], i64 0
; IND-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
+; IND-NEXT: [[TMP0:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND]], <2 x i32> <i32 1, i32 2>
; IND-NEXT: [[TMP2:%.*]] = mul nsw <2 x i32> [[BROADCAST_SPLAT]], [[TMP0]]
; IND-NEXT: [[SEXT:%.*]] = shl i64 [[INDEX]], 32
; IND-NEXT: [[TMP3:%.*]] = ashr exact i64 [[SEXT]], 30
@@ -5721,11 +5721,11 @@ define void @pr52460_first_order_recurrence_truncated_iv(ptr noalias %src, ptr %
; UNROLL-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i32> [ <i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[STEP_ADD:%.*]], [[VECTOR_BODY]] ]
; UNROLL-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; UNROLL-NEXT: [[STEP_ADD]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
-; UNROLL-NEXT: [[TMP0:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND]], <2 x i32> <i32 1, i32 2>
-; UNROLL-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[VEC_IND]], <2 x i32> [[STEP_ADD]], <2 x i32> <i32 1, i32 2>
; UNROLL-NEXT: [[TMP2:%.*]] = load i32, ptr [[SRC:%.*]], align 4
; UNROLL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP2]], i64 0
; UNROLL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
+; UNROLL-NEXT: [[TMP0:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND]], <2 x i32> <i32 1, i32 2>
+; UNROLL-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[VEC_IND]], <2 x i32> [[STEP_ADD]], <2 x i32> <i32 1, i32 2>
; UNROLL-NEXT: [[TMP3:%.*]] = mul nsw <2 x i32> [[BROADCAST_SPLAT]], [[TMP0]]
; UNROLL-NEXT: [[TMP4:%.*]] = mul nsw <2 x i32> [[BROADCAST_SPLAT]], [[TMP1]]
; UNROLL-NEXT: [[SEXT:%.*]] = shl i64 [[INDEX]], 32
@@ -5756,11 +5756,11 @@ define void @pr52460_first_order_recurrence_truncated_iv(ptr noalias %src, ptr %
; UNROLL-NO-IC-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; UNROLL-NO-IC-NEXT: [[STEP_ADD]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
; UNROLL-NO-IC-NEXT: [[TMP0:%.*]] = trunc i64 [[INDEX]] to i32
-; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND]], <2 x i32> <i32 1, i32 2>
-; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = shufflevector <2 x i32> [[VEC_IND]], <2 x i32> [[STEP_ADD]], <2 x i32> <i32 1, i32 2>
; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = load i32, ptr [[SRC:%.*]], align 4
; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i64 0
; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> poison, <2 x i32> zeroinitializer
+; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[VECTOR_RECUR]], <2 x i32> [[VEC_IND]], <2 x i32> <i32 1, i32 2>
+; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = shufflevector <2 x i32> [[VEC_IND]], <2 x i32> [[STEP_ADD]], <2 x i32> <i32 1, i32 2>
; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = mul nsw <2 x i32> [[BROADCAST_SPLAT]], [[TMP2]]
; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = mul nsw <2 x i32> [[BROADCAST_SPLAT]], [[TMP3]]
; UNROLL-NO-IC-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[DST:%.*]], i32 [[TMP0]]
@@ -5788,11 +5788,11 @@ define void @pr52460_first_order_recurrence_truncated_iv(ptr noalias %src, ptr %
; INTERLEAVE-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[STEP_ADD:%.*]], [[VECTOR_BODY]] ]
; INTERLEAVE-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; INTERLEAVE-NEXT: [[STEP_ADD]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
-; INTERLEAVE-NEXT: [[TMP0:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[VEC_IND]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; INTERLEAVE-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[VEC_IND]], <4 x i32> [[STEP_ADD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; INTERLEAVE-NEXT: [[TMP2:%.*]] = load i32, ptr [[SRC:%.*]], align 4
; INTERLEAVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP2]], i64 0
; INTERLEAVE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; INTERLEAVE-NEXT: [[TMP0:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[VEC_IND]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; INTERLEAVE-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[VEC_IND]], <4 x i32> [[STEP_ADD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; INTERLEAVE-NEXT: [[TMP3:%.*]] = mul nsw <4 x i32> [[BROADCAST_SPLAT]], [[TMP0]]
; INTERLEAVE-NEXT: [[TMP4:%.*]] = mul nsw <4 x i32> [[BROADCAST_SPLAT]], [[TMP1]]
; INTERLEAVE-NEXT: [[SEXT:%.*]] = shl i64 [[INDEX]], 32
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
index ec85f7eb8748e..45b693e455ea9 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses.ll
@@ -1475,9 +1475,9 @@ define void @PR34743(ptr %a, ptr %b, i64 %n) {
; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i16>, ptr [[TMP7]], align 4, !alias.scope [[META27:![0-9]+]]
; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i16> [[WIDE_VEC]], <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
; CHECK-NEXT: [[STRIDED_VEC4]] = shufflevector <8 x i16> [[WIDE_VEC]], <8 x i16> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[TMP10:%.*]] = sext <4 x i16> [[STRIDED_VEC]] to <4 x i32>
; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i16> [[VECTOR_RECUR]], <4 x i16> [[STRIDED_VEC4]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; CHECK-NEXT: [[TMP9:%.*]] = sext <4 x i16> [[TMP8]] to <4 x i32>
-; CHECK-NEXT: [[TMP10:%.*]] = sext <4 x i16> [[STRIDED_VEC]] to <4 x i32>
; CHECK-NEXT: [[TMP11:%.*]] = sext <4 x i16> [[STRIDED_VEC4]] to <4 x i32>
; CHECK-NEXT: [[TMP12:%.*]] = mul nsw <4 x i32> [[TMP9]], [[TMP10]]
; CHECK-NEXT: [[TMP13:%.*]] = mul nsw <4 x i32> [[TMP12]], [[TMP11]]
>From 4cb96f827d89af3a87c74ac3b0838c0d6bfab730 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 16 Apr 2026 16:09:15 +0100
Subject: [PATCH 2/5] !fixup address comments, thanks
---
.../Vectorize/VPlanConstruction.cpp | 92 ++++++++++---------
.../Transforms/Vectorize/VPlanTransforms.cpp | 20 ++--
...an-vp-intrinsics-fixed-order-recurrence.ll | 3 +-
3 files changed, 61 insertions(+), 54 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index b6fe5b52f9531..b2557a5b72921 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -854,6 +854,53 @@ static bool hoistPreviousBeforeFORUsers(VPFirstOrderRecurrencePHIRecipe *FOR,
return true;
}
+// Sink users of fixed-order recurrences past or hoist before the recipe
+// defining the previous value, introduce FirstOrderRecurrenceSplice
+// VPInstructions, and replace FOR uses. Return false, if hoisting or sinking
+// fails.
+static bool tryToSinkOrHoistRecurrenceUsers(VPBasicBlock *HeaderVPBB,
+ VPDominatorTree &VPDT) {
+ for (VPRecipeBase &R : HeaderVPBB->phis()) {
+ auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R);
+ if (!FOR)
+ continue;
+
+ // Follow through FOR phi chains to find the actual Previous recipe.
+ // Fixed-order recurrences do not contain cycles, so this loop is
+ // guaranteed to terminate.
+ SmallPtrSet<VPFirstOrderRecurrencePHIRecipe *, 4> SeenPhis;
+ VPRecipeBase *Previous = FOR->getBackedgeValue()->getDefiningRecipe();
+ while (auto *PrevPhi =
+ dyn_cast_or_null<VPFirstOrderRecurrencePHIRecipe>(Previous)) {
+ assert(PrevPhi->getParent() == FOR->getParent());
+ assert(SeenPhis.insert(PrevPhi).second);
+ Previous = PrevPhi->getBackedgeValue()->getDefiningRecipe();
+ }
+
+ // Sink FOR users after Previous or hoist Previous before FOR users.
+ if (!sinkRecurrenceUsersAfterPrevious(FOR, Previous, VPDT) &&
+ !hoistPreviousBeforeFORUsers(FOR, Previous, VPDT))
+ return false;
+
+ // Create FirstOrderRecurrenceSplice and replace FOR uses.
+ VPBasicBlock *InsertBlock =
+ Previous ? Previous->getParent() : FOR->getParent();
+ auto InsertPt = (!Previous || isa<VPHeaderPHIRecipe>(Previous))
+ ? InsertBlock->getFirstNonPhi()
+ : std::next(Previous->getIterator());
+ VPBuilder LoopBuilder(InsertBlock, InsertPt);
+ ;
+ auto *RecurSplice =
+ LoopBuilder.createNaryOp(VPInstruction::FirstOrderRecurrenceSplice,
+ {FOR, FOR->getBackedgeValue()});
+ FOR->replaceUsesWithIf(RecurSplice, [RecurSplice](VPUser &U, unsigned) {
+ return &U != RecurSplice;
+ });
+ }
+
+ return true;
+}
+
bool VPlanTransforms::createHeaderPhiRecipes(
VPlan &Plan, PredicatedScalarEvolution &PSE, Loop &OrigLoop,
const MapVector<PHINode *, InductionDescriptor> &Inductions,
@@ -919,49 +966,8 @@ bool VPlanTransforms::createHeaderPhiRecipes(
PhiR->eraseFromParent();
}
- // Sink users of fixed-order recurrences past the recipe defining the
- // previous value, introduce FirstOrderRecurrenceSplice VPInstructions, and
- // replace FOR uses. At VPlan0, the VPlan mirrors the scalar IR, so the
- // legality check (sink/hoist) is guaranteed to succeed by
- // isFixedOrderRecurrence.
- VPBuilder LoopBuilder;
- for (VPRecipeBase &R : HeaderVPBB->phis()) {
- auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R);
- if (!FOR)
- continue;
-
- // Follow through FOR phi chains to find the actual Previous recipe.
- // Fixed-order recurrences do not contain cycles, so this loop is
- // guaranteed to terminate.
- SmallPtrSet<VPFirstOrderRecurrencePHIRecipe *, 4> SeenPhis;
- VPRecipeBase *Previous = FOR->getBackedgeValue()->getDefiningRecipe();
- while (auto *PrevPhi =
- dyn_cast_or_null<VPFirstOrderRecurrencePHIRecipe>(Previous)) {
- assert(PrevPhi->getParent() == FOR->getParent());
- assert(SeenPhis.insert(PrevPhi).second);
- Previous = PrevPhi->getBackedgeValue()->getDefiningRecipe();
- }
-
- // Sink FOR users after Previous or hoist Previous before FOR users.
- if (!sinkRecurrenceUsersAfterPrevious(FOR, Previous, VPDT) &&
- !hoistPreviousBeforeFORUsers(FOR, Previous, VPDT))
- return false;
-
- // Create FirstOrderRecurrenceSplice and replace FOR uses.
- VPBasicBlock *InsertBlock =
- Previous ? Previous->getParent() : FOR->getParent();
- auto InsertPt = (!Previous || isa<VPHeaderPHIRecipe>(Previous))
- ? InsertBlock->getFirstNonPhi()
- : std::next(Previous->getIterator());
- LoopBuilder.setInsertPoint(InsertBlock, InsertPt);
-
- auto *RecurSplice =
- LoopBuilder.createNaryOp(VPInstruction::FirstOrderRecurrenceSplice,
- {FOR, FOR->getBackedgeValue()});
- FOR->replaceUsesWithIf(RecurSplice, [RecurSplice](VPUser &U, unsigned) {
- return &U != RecurSplice;
- });
- }
+ if (!tryToSinkOrHoistRecurrenceUsers(HeaderVPBB, VPDT))
+ return false;
for (const auto &[HeaderPhiR, ScalarPhiR] :
getMatchingPhisForScalarLoop(HeaderVPBB, Plan.getScalarPreheader())) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index d48e681172c99..5487c340519da 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5384,15 +5384,6 @@ void VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers(VPlan &Plan,
if (!FOR)
continue;
- // For VF vscale x 1, if vscale = 1, we are unable to extract the
- // penultimate value of the recurrence. Instead we rely on the existing
- // extract of the last element from the result of
- // VPInstruction::FirstOrderRecurrenceSplice.
- // TODO: Consider vscale_range info and UF.
- if (LoopVectorizationPlanner::getDecisionAndClampRange(IsScalableOne,
- Range))
- return;
-
assert(VectorRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
"Cannot handle loops with uncountable early exits");
@@ -5407,6 +5398,17 @@ void VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers(VPlan &Plan,
assert(It != FOR->users().end() && "expected FirstOrderRecurrenceSplice");
auto *RecurSplice = cast<VPInstruction>(*It);
+ // For VF vscale x 1, if vscale = 1, we are unable to extract the
+ // penultimate value of the recurrence. Instead we rely on the existing
+ // extract of the last element from the result of
+ // VPInstruction::FirstOrderRecurrenceSplice.
+ // TODO: Consider vscale_range info and UF.
+ if (any_of(RecurSplice->users(),
+ [](VPUser *U) { return !cast<VPRecipeBase>(U)->getRegion(); }) &&
+ LoopVectorizationPlanner::getDecisionAndClampRange(IsScalableOne,
+ Range))
+ return;
+
// This is the second phase of vectorizing first-order recurrences, creating
// extracts for users outside the loop. An overview of the transformation is
// described below. Suppose we have the following loop with some use after
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll
index 389264d6579d1..f87b3326808fd 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/RISCV/vplan-vp-intrinsics-fixed-order-recurrence.ll
@@ -5,9 +5,8 @@
define void @first_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
; IF-EVL: VPlan 'Initial VPlan for VF={1},UF>=1'
; IF-EVL-NOT: CURRENT-ITERATION-PHI
-; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1},UF={1}'
;
-; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 2,vscale x 4},UF={1}' {
+; IF-EVL: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
; IF-EVL-NEXT: Live-in vp<[[VF:%[0-9]+]]> = VF
; IF-EVL-NEXT: Live-in vp<[[VFUF:%[0-9]+]]> = VF * UF
; IF-EVL-NEXT: Live-in vp<[[VTC:%[0-9]+]]> = vector-trip-count
>From e0380c45fc1477e90518eb6a3c2f720b0d11bdb6 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 22 Apr 2026 10:17:30 +0100
Subject: [PATCH 3/5] !fixup after merge
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 15 ++++++---------
.../Transforms/Vectorize/VPlanConstruction.cpp | 9 ++++-----
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 11 ++++-------
.../LoopVectorize/VPlan/vplan-print-after-all.ll | 2 +-
4 files changed, 15 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 690eb79103dad..9e0dc17284508 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7634,10 +7634,11 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE, &LVer);
// Create recipes for header phis.
- if (!RUN_VPLAN_PASS(VPlanTransforms::createHeaderPhiRecipes,
- *VPlan0, PSE, *OrigLoop, Legal->getInductionVars(),
- Legal->getReductionVars(), Legal->getFixedOrderRecurrences(),
- CM.getInLoopReductions(), Hints.allowReordering()))
+ if (!RUN_VPLAN_PASS(VPlanTransforms::createHeaderPhiRecipes, *VPlan0, PSE,
+ *OrigLoop, Legal->getInductionVars(),
+ Legal->getReductionVars(),
+ Legal->getFixedOrderRecurrences(),
+ CM.getInLoopReductions(), Hints.allowReordering()))
return;
RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, *VPlan0);
@@ -7846,14 +7847,10 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
"entry block must be set to a VPRegionBlock having a non-empty entry "
"VPBasicBlock");
-<<<<<<< HEAD
// TODO: We can't call runPass on these transforms yet, due to verifier
// failures.
- VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers(*Plan, Range);
-=======
- RUN_VPLAN_PASS(VPlanTransforms::addExitUsersForFirstOrderRecurrences, *Plan,
+ RUN_VPLAN_PASS(VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers, *Plan,
Range);
->>>>>>> origin/main
// ---------------------------------------------------------------------------
// Transform initial VPlan: Apply previously taken decisions, in order, to
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index bff8e11d951ce..7d34ff08e4fb8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -873,10 +873,10 @@ static bool hoistPreviousBeforeFORUsers(VPFirstOrderRecurrencePHIRecipe *FOR,
return true;
}
-// Sink users of fixed-order recurrences past or hoist before the recipe
-// defining the previous value, introduce FirstOrderRecurrenceSplice
-// VPInstructions, and replace FOR uses. Return false, if hoisting or sinking
-// fails.
+/// Sink users of fixed-order recurrences past or hoist before the recipe
+/// defining the previous value, introduce FirstOrderRecurrenceSplice
+/// VPInstructions, and replace FOR uses. Returns false if hoisting or sinking
+/// fails.
static bool tryToSinkOrHoistRecurrenceUsers(VPBasicBlock *HeaderVPBB,
VPDominatorTree &VPDT) {
for (VPRecipeBase &R : HeaderVPBB->phis()) {
@@ -908,7 +908,6 @@ static bool tryToSinkOrHoistRecurrenceUsers(VPBasicBlock *HeaderVPBB,
? InsertBlock->getFirstNonPhi()
: std::next(Previous->getIterator());
VPBuilder LoopBuilder(InsertBlock, InsertPt);
- ;
auto *RecurSplice =
LoopBuilder.createNaryOp(VPInstruction::FirstOrderRecurrenceSplice,
{FOR, FOR->getBackedgeValue()});
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index dee79ce71ca3f..a883685f875a1 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5405,13 +5405,9 @@ void VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers(VPlan &Plan,
// Find the existing splice for this FOR, created in
// createHeaderPhiRecipes. All uses of FOR have already been replaced with
// RecurSplice there; only RecurSplice itself still references FOR.
- auto It = find_if(FOR->users(), [&](VPUser *U) {
- return match(U,
- m_VPInstruction<VPInstruction::FirstOrderRecurrenceSplice>(
- m_Specific(FOR), m_VPValue()));
- });
- assert(It != FOR->users().end() && "expected FirstOrderRecurrenceSplice");
- auto *RecurSplice = cast<VPInstruction>(*It);
+ auto *RecurSplice =
+ vputils::findUserOf<VPInstruction::FirstOrderRecurrenceSplice>(FOR);
+ assert(RecurSplice && "expected FirstOrderRecurrenceSplice");
// For VF vscale x 1, if vscale = 1, we are unable to extract the
// penultimate value of the recurrence. Instead we rely on the existing
@@ -5515,6 +5511,7 @@ void VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers(VPlan &Plan,
VPValue *Cmp = B.createICmp(CmpInst::ICMP_EQ, LastActiveLane, Zero);
VPValue *Sel = B.createSelect(Cmp, LastPrevIter, PenultimateLastIter);
ExtractR->replaceAllUsesWith(Sel);
+ continue;
}
if (!match(&R, m_ExtractLastLaneOfLastPart(m_Specific(RecurSplice))))
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
index 0df8b4d2b9674..fd97dd84cab45 100644
--- a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-print-after-all.ll
@@ -12,7 +12,7 @@
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::introduceMasksAndLinearize
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::createInLoopReductionRecipes
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::makeMemOpWideningDecisions
-; CHECK: VPlan for loop in 'foo' after VPlanTransforms::addExitUsersForFirstOrderRecurrences
+; CHECK: VPlan for loop in 'foo' after VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::clearReductionWrapFlags
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::optimizeFindIVReductions
; CHECK: VPlan for loop in 'foo' after VPlanTransforms::handleMultiUseReductions
>From 830cce0404b9819475eef47ddbed7a4368bc5ace Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 22 Apr 2026 21:12:06 +0100
Subject: [PATCH 4/5] !fixup address latest comments, thanks!
---
.../Vectorize/VPlanConstruction.cpp | 24 +++++++++----------
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 7d34ff08e4fb8..d96c58d57b2b4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -717,17 +717,11 @@ createWidenInductionRecipe(PHINode *Phi, VPPhi *PhiR, VPIRValue *Start,
return WideIV;
}
-/// Try to sink users of \p FOR after \p Previous. \returns true if sinking
-/// succeeded or was not necessary.
-static bool
+/// Try to sink users of \p FOR after \p Previous. \returns true if sinking
+/// -/// succeeded or was not necessary, and false otherwise.
sinkRecurrenceUsersAfterPrevious(VPFirstOrderRecurrencePHIRecipe *FOR,
VPRecipeBase *Previous,
VPDominatorTree &VPDT) {
- // If Previous is a live-in (no defining recipe), it naturally dominates all
- // recipes in the loop, so no sinking is needed.
- if (!Previous)
- return true;
-
// Collect recipes that need sinking.
SmallVector<VPRecipeBase *> WorkList;
SmallPtrSet<VPRecipeBase *, 8> Seen;
@@ -780,7 +774,7 @@ sinkRecurrenceUsersAfterPrevious(VPFirstOrderRecurrencePHIRecipe *FOR,
}
/// Try to hoist \p Previous and its operands before all users of \p FOR.
-/// \returns true if hoisting succeeded or was not necessary.
+/// \returns true if hoisting succeeded or was not necessary, and false otherwise.
static bool hoistPreviousBeforeFORUsers(VPFirstOrderRecurrencePHIRecipe *FOR,
VPRecipeBase *Previous,
VPDominatorTree &VPDT) {
@@ -896,9 +890,15 @@ static bool tryToSinkOrHoistRecurrenceUsers(VPBasicBlock *HeaderVPBB,
Previous = PrevPhi->getBackedgeValue()->getDefiningRecipe();
}
- // Sink FOR users after Previous or hoist Previous before FOR users.
- if (!sinkRecurrenceUsersAfterPrevious(FOR, Previous, VPDT) &&
- !hoistPreviousBeforeFORUsers(FOR, Previous, VPDT))
+ // If Previous is a live-in (no defining recipe), it naturally dominates all
+ // recipes in the loop, so no sinking is needed.
+ if (!Previous)
+ return true;
+
+
+ // Sink FOR users after Previous or hoist Previous before FOR users, if i is a recipe.
+ if (Previous && (!sinkRecurrenceUsersAfterPrevious(FOR, Previous, VPDT) &&
+ !hoistPreviousBeforeFORUsers(FOR, Previous, VPDT)))
return false;
// Create FirstOrderRecurrenceSplice and replace FOR uses.
>From d873a27441cab59ea7438a557464dfca44328167 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Thu, 23 Apr 2026 11:57:08 +0100
Subject: [PATCH 5/5] !fixup remove code handling Previous not being a recipe
---
.../Vectorize/VPlanConstruction.cpp | 21 +++++++------------
1 file changed, 8 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 49fea48bf9f26..574f14169faa3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -775,7 +775,8 @@ sinkRecurrenceUsersAfterPrevious(VPFirstOrderRecurrencePHIRecipe *FOR,
}
/// Try to hoist \p Previous and its operands before all users of \p FOR.
-/// \returns true if hoisting succeeded or was not necessary, and false otherwise.
+/// \returns true if hoisting succeeded or was not necessary, and false
+/// otherwise.
static bool hoistPreviousBeforeFORUsers(VPFirstOrderRecurrencePHIRecipe *FOR,
VPRecipeBase *Previous,
VPDominatorTree &VPDT) {
@@ -891,21 +892,15 @@ static bool tryToSinkOrHoistRecurrenceUsers(VPBasicBlock *HeaderVPBB,
Previous = PrevPhi->getBackedgeValue()->getDefiningRecipe();
}
- // If Previous is a live-in (no defining recipe), it naturally dominates all
- // recipes in the loop, so no sinking is needed.
- if (!Previous)
- return true;
-
-
- // Sink FOR users after Previous or hoist Previous before FOR users, if i is a recipe.
- if (Previous && (!sinkRecurrenceUsersAfterPrevious(FOR, Previous, VPDT) &&
- !hoistPreviousBeforeFORUsers(FOR, Previous, VPDT)))
+ assert(Previous && "Previous must be a recipe");
+ // Sink FOR users after Previous or hoist Previous before FOR users.
+ if (!sinkRecurrenceUsersAfterPrevious(FOR, Previous, VPDT) &&
+ !hoistPreviousBeforeFORUsers(FOR, Previous, VPDT))
return false;
// Create FirstOrderRecurrenceSplice and replace FOR uses.
- VPBasicBlock *InsertBlock =
- Previous ? Previous->getParent() : FOR->getParent();
- auto InsertPt = (!Previous || isa<VPHeaderPHIRecipe>(Previous))
+ VPBasicBlock *InsertBlock = Previous->getParent();
+ auto InsertPt = isa<VPHeaderPHIRecipe>(Previous)
? InsertBlock->getFirstNonPhi()
: std::next(Previous->getIterator());
VPBuilder LoopBuilder(InsertBlock, InsertPt);
More information about the llvm-commits
mailing list