[llvm] 4fd8dbc - [LV] Move code to prepare VPlan for epilogue vector loop to helper (NFC)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Mon Dec 9 13:58:26 PST 2024
Author: Florian Hahn
Date: 2024-12-09T21:56:10Z
New Revision: 4fd8dbc1842130a25b1292da17828bb6f3572d9b
URL: https://github.com/llvm/llvm-project/commit/4fd8dbc1842130a25b1292da17828bb6f3572d9b
DIFF: https://github.com/llvm/llvm-project/commit/4fd8dbc1842130a25b1292da17828bb6f3572d9b.diff
LOG: [LV] Move code to prepare VPlan for epilogue vector loop to helper (NFC)
Move code to prepare the VPlan for the epilogue vector loop to a helper
to reduce size and complexity of processLoop.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 257fa63fcb29d8..16b84bf6a7bd51 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9989,6 +9989,77 @@ LoopVectorizePass::LoopVectorizePass(LoopVectorizeOptions Opts)
VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced ||
!EnableLoopVectorization) {}
+/// Prepare \p Plan for vectorizing the epilogue loop. That is, re-use expanded
+/// SCEVs from \p ExpandedSCEVs and set resume values for header recipes.
+static void
+preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
+ const SCEV2ValueTy &ExpandedSCEVs) {
+ VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
+ VPBasicBlock *Header = VectorLoop->getEntryBasicBlock();
+ Header->setName("vec.epilog.vector.body");
+
+ // Re-use the trip count and steps expanded for the main loop, as
+ // skeleton creation needs it as a value that dominates both the scalar
+ // and vector epilogue loops
+ // TODO: This is a workaround needed for epilogue vectorization and it
+ // should be removed once induction resume value creation is done
+ // directly in VPlan.
+ for (auto &R : make_early_inc_range(*Plan.getPreheader())) {
+ auto *ExpandR = dyn_cast<VPExpandSCEVRecipe>(&R);
+ if (!ExpandR)
+ continue;
+ auto *ExpandedVal =
+ Plan.getOrAddLiveIn(ExpandedSCEVs.find(ExpandR->getSCEV())->second);
+ ExpandR->replaceAllUsesWith(ExpandedVal);
+ if (Plan.getTripCount() == ExpandR)
+ Plan.resetTripCount(ExpandedVal);
+ ExpandR->eraseFromParent();
+ }
+
+ // Ensure that the start values for any VPWidenIntOrFpInductionRecipe,
+ // VPWidenPointerInductionRecipe and VPReductionPHIRecipes are updated
+ // before vectorizing the epilogue loop.
+ for (VPRecipeBase &R : Header->phis()) {
+ if (isa<VPCanonicalIVPHIRecipe>(&R))
+ continue;
+
+ Value *ResumeV = nullptr;
+ // TODO: Move setting of resume values to prepareToExecute.
+ if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R)) {
+ ResumeV = cast<PHINode>(ReductionPhi->getUnderlyingInstr())
+ ->getIncomingValueForBlock(L->getLoopPreheader());
+ const RecurrenceDescriptor &RdxDesc =
+ ReductionPhi->getRecurrenceDescriptor();
+ RecurKind RK = RdxDesc.getRecurrenceKind();
+ if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
+ // VPReductionPHIRecipes for AnyOf reductions expect a boolean as
+ // start value; compare the final value from the main vector loop
+ // to the start value.
+ IRBuilder<> Builder(
+ cast<Instruction>(ResumeV)->getParent()->getFirstNonPHI());
+ ResumeV =
+ Builder.CreateICmpNE(ResumeV, RdxDesc.getRecurrenceStartValue());
+ }
+ } else {
+ // Retrieve the induction resume values for wide inductions from
+ // their original phi nodes in the scalar loop.
+ PHINode *IndPhi = nullptr;
+ if (auto *Ind = dyn_cast<VPWidenPointerInductionRecipe>(&R)) {
+ IndPhi = cast<PHINode>(Ind->getUnderlyingValue());
+ } else {
+ auto *WidenInd = cast<VPWidenIntOrFpInductionRecipe>(&R);
+ IndPhi = WidenInd->getPHINode();
+ }
+ // Hook up to the PHINode generated by a ResumePhi recipe of main
+ // loop VPlan, which feeds the scalar loop.
+ ResumeV = IndPhi->getIncomingValueForBlock(L->getLoopPreheader());
+ }
+ assert(ResumeV && "Must have a resume value");
+ VPValue *StartVal = Plan.getOrAddLiveIn(ResumeV);
+ cast<VPHeaderPHIRecipe>(&R)->setStartValue(StartVal);
+ }
+}
+
bool LoopVectorizePass::processLoop(Loop *L) {
assert((EnableVPlanNativePath || L->isInnermost()) &&
"VPlan-native path is not enabled. Only process inner loops.");
@@ -10353,72 +10424,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
EpilogueVectorizerEpilogueLoop EpilogILV(L, PSE, LI, DT, TLI, TTI, AC,
ORE, EPI, &LVL, &CM, BFI, PSI,
Checks, BestEpiPlan);
-
- VPRegionBlock *VectorLoop = BestEpiPlan.getVectorLoopRegion();
- VPBasicBlock *Header = VectorLoop->getEntryBasicBlock();
- Header->setName("vec.epilog.vector.body");
-
- // Re-use the trip count and steps expanded for the main loop, as
- // skeleton creation needs it as a value that dominates both the scalar
- // and vector epilogue loops
- // TODO: This is a workaround needed for epilogue vectorization and it
- // should be removed once induction resume value creation is done
- // directly in VPlan.
EpilogILV.setTripCount(MainILV.getTripCount());
- for (auto &R : make_early_inc_range(*BestEpiPlan.getPreheader())) {
- auto *ExpandR = dyn_cast<VPExpandSCEVRecipe>(&R);
- if (!ExpandR)
- continue;
- auto *ExpandedVal = BestEpiPlan.getOrAddLiveIn(
- ExpandedSCEVs.find(ExpandR->getSCEV())->second);
- ExpandR->replaceAllUsesWith(ExpandedVal);
- if (BestEpiPlan.getTripCount() == ExpandR)
- BestEpiPlan.resetTripCount(ExpandedVal);
- ExpandR->eraseFromParent();
- }
-
- // Ensure that the start values for any VPWidenIntOrFpInductionRecipe,
- // VPWidenPointerInductionRecipe and VPReductionPHIRecipes are updated
- // before vectorizing the epilogue loop.
- for (VPRecipeBase &R : Header->phis()) {
- if (isa<VPCanonicalIVPHIRecipe>(&R))
- continue;
-
- Value *ResumeV = nullptr;
- // TODO: Move setting of resume values to prepareToExecute.
- if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R)) {
- ResumeV = cast<PHINode>(ReductionPhi->getUnderlyingInstr())
- ->getIncomingValueForBlock(L->getLoopPreheader());
- const RecurrenceDescriptor &RdxDesc =
- ReductionPhi->getRecurrenceDescriptor();
- RecurKind RK = RdxDesc.getRecurrenceKind();
- if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
- // VPReductionPHIRecipes for AnyOf reductions expect a boolean as
- // start value; compare the final value from the main vector loop
- // to the start value.
- IRBuilder<> Builder(
- cast<Instruction>(ResumeV)->getParent()->getFirstNonPHI());
- ResumeV = Builder.CreateICmpNE(ResumeV,
- RdxDesc.getRecurrenceStartValue());
- }
- } else {
- // Retrieve the induction resume values for wide inductions from
- // their original phi nodes in the scalar loop.
- PHINode *IndPhi = nullptr;
- if (auto *Ind = dyn_cast<VPWidenPointerInductionRecipe>(&R)) {
- IndPhi = cast<PHINode>(Ind->getUnderlyingValue());
- } else {
- auto *WidenInd = cast<VPWidenIntOrFpInductionRecipe>(&R);
- IndPhi = WidenInd->getPHINode();
- }
- // Hook up to the PHINode generated by a ResumePhi recipe of main
- // loop VPlan, which feeds the scalar loop.
- ResumeV = IndPhi->getIncomingValueForBlock(L->getLoopPreheader());
- }
- assert(ResumeV && "Must have a resume value");
- VPValue *StartVal = BestEpiPlan.getOrAddLiveIn(ResumeV);
- cast<VPHeaderPHIRecipe>(&R)->setStartValue(StartVal);
- }
+ preparePlanForEpilogueVectorLoop(BestEpiPlan, L, ExpandedSCEVs);
assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&
"DT not preserved correctly");
More information about the llvm-commits
mailing list