[llvm] bf4486e - [LV] Move fixing reduction resumes for epilogue out of executePlan.(NFC)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 1 11:41:57 PDT 2025
Author: Florian Hahn
Date: 2025-09-01T19:39:00+01:00
New Revision: bf4486eb29c3009ddf68968b021c6fd98e3c2d52
URL: https://github.com/llvm/llvm-project/commit/bf4486eb29c3009ddf68968b021c6fd98e3c2d52
DIFF: https://github.com/llvm/llvm-project/commit/bf4486eb29c3009ddf68968b021c6fd98e3c2d52.diff
LOG: [LV] Move fixing reduction resumes for epilogue out of executePlan.(NFC)
Move fixing up reduction resume values out of the general ::executePlan
and perform it together with updating induction resume values.
This also allows moving additional bypass block handling to
EpilogueVectorizerEpilogueLoop.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index a02f7cd3393bf..dfef6eb300230 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -532,14 +532,6 @@ class InnerLoopVectorizer {
/// count of the original loop for both main loop and epilogue vectorization.
void setTripCount(Value *TC) { TripCount = TC; }
- /// Return the additional bypass block which targets the scalar loop by
- /// skipping the epilogue loop after completing the main loop.
- BasicBlock *getAdditionalBypassBlock() const {
- assert(AdditionalBypassBlock &&
- "Trying to access AdditionalBypassBlock but it has not been set");
- return AdditionalBypassBlock;
- }
-
protected:
friend class LoopVectorizationPlanner;
@@ -602,11 +594,6 @@ class InnerLoopVectorizer {
/// for cleaning the checks, if vectorization turns out unprofitable.
GeneratedRTChecks &RTChecks;
- /// The additional bypass block which conditionally skips over the epilogue
- /// loop after executing the main loop. Needed to resume inductions and
- /// reductions during epilogue vectorization.
- BasicBlock *AdditionalBypassBlock = nullptr;
-
VPlan &Plan;
/// The vector preheader block of \p Plan, used as target for check blocks
@@ -711,6 +698,11 @@ class EpilogueVectorizerMainLoop : public InnerLoopAndEpilogueVectorizer {
// vectorization of *epilogue* loops in the process of vectorizing loops and
// their epilogues.
class EpilogueVectorizerEpilogueLoop : public InnerLoopAndEpilogueVectorizer {
+ /// The additional bypass block which conditionally skips over the epilogue
+ /// loop after executing the main loop. Needed to resume inductions and
+ /// reductions during epilogue vectorization.
+ BasicBlock *AdditionalBypassBlock = nullptr;
+
public:
EpilogueVectorizerEpilogueLoop(
Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI,
@@ -727,6 +719,14 @@ class EpilogueVectorizerEpilogueLoop : public InnerLoopAndEpilogueVectorizer {
/// *epilogue loop* strategy (i.e., the second pass of VPlan execution).
BasicBlock *createVectorizedLoopSkeleton() final;
+ /// Return the additional bypass block which targets the scalar loop by
+ /// skipping the epilogue loop after completing the main loop.
+ BasicBlock *getAdditionalBypassBlock() const {
+ assert(AdditionalBypassBlock &&
+ "Trying to access AdditionalBypassBlock but it has not been set");
+ return AdditionalBypassBlock;
+ }
+
protected:
/// Emits an iteration count bypass check after the main vector loop has
/// finished to see if there are any iterations left to execute by either
@@ -7139,7 +7139,7 @@ static Value *getStartValueFromReductionResult(VPInstruction *RdxResult) {
// epilog loop, fix the reduction's scalar PHI node by adding the incoming value
// from the main vector loop.
static void fixReductionScalarResumeWhenVectorizingEpilog(
- VPPhi *EpiResumePhiR, VPTransformState &State, BasicBlock *BypassBlock) {
+ VPPhi *EpiResumePhiR, PHINode &EpiResumePhi, BasicBlock *BypassBlock) {
// Get the VPInstruction computing the reduction result in the middle block.
// The first operand may not be from the middle block if it is not connected
// to the scalar preheader. In that case, there's nothing to fix.
@@ -7194,8 +7194,7 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
// When fixing reductions in the epilogue loop we should already have
// created a bc.merge.rdx Phi after the main vector body. Ensure that we carry
// over the incoming values correctly.
- auto *EpiResumePhi = cast<PHINode>(State.get(EpiResumePhiR, true));
- EpiResumePhi->setIncomingValueForBlock(
+ EpiResumePhi.setIncomingValueForBlock(
BypassBlock, MainResumePhi->getIncomingValueForBlock(BypassBlock));
}
@@ -7306,31 +7305,6 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
BestVPlan.execute(&State);
- // 2.5 When vectorizing the epilogue, fix reduction resume values from the
- // additional bypass block.
- if (VectorizingEpilogue) {
- assert(!BestVPlan.hasEarlyExit() &&
- "Epilogue vectorisation not yet supported with early exits");
- BasicBlock *PH = OrigLoop->getLoopPreheader();
- BasicBlock *BypassBlock = ILV.getAdditionalBypassBlock();
- for (auto *Pred : predecessors(PH)) {
- for (PHINode &Phi : PH->phis()) {
- if (Phi.getBasicBlockIndex(Pred) != -1)
- continue;
- Phi.addIncoming(Phi.getIncomingValueForBlock(BypassBlock), Pred);
- }
- }
- VPBasicBlock *ScalarPH = BestVPlan.getScalarPreheader();
- if (ScalarPH->hasPredecessors()) {
- // If ScalarPH has predecessors, we may need to update its reduction
- // resume values.
- for (VPRecipeBase &R : ScalarPH->phis()) {
- fixReductionScalarResumeWhenVectorizingEpilog(cast<VPPhi>(&R), State,
- BypassBlock);
- }
- }
- }
-
// 2.6. Maintain Loop Hints
// Keep all loop hints from the original loop on the vector loop (we'll
// replace the vectorizer-specific hints below).
@@ -9851,6 +9825,43 @@ static Value *createInductionAdditionalBypassValues(
return EndValueFromAdditionalBypass;
}
+static void fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L,
+ VPlan &BestEpiPlan,
+ LoopVectorizationLegality &LVL,
+ const SCEV2ValueTy &ExpandedSCEVs,
+ Value *MainVectorTripCount) {
+ // Fix reduction resume values from the additional bypass block.
+ BasicBlock *PH = L->getLoopPreheader();
+ for (auto *Pred : predecessors(PH)) {
+ for (PHINode &Phi : PH->phis()) {
+ if (Phi.getBasicBlockIndex(Pred) != -1)
+ continue;
+ Phi.addIncoming(Phi.getIncomingValueForBlock(BypassBlock), Pred);
+ }
+ }
+ auto *ScalarPH = cast<VPIRBasicBlock>(BestEpiPlan.getScalarPreheader());
+ if (ScalarPH->hasPredecessors()) {
+ // If ScalarPH has predecessors, we may need to update its reduction
+ // resume values.
+ for (const auto &[R, IRPhi] :
+ zip(ScalarPH->phis(), ScalarPH->getIRBasicBlock()->phis())) {
+ fixReductionScalarResumeWhenVectorizingEpilog(cast<VPPhi>(&R), IRPhi,
+ BypassBlock);
+ }
+ }
+
+ // Fix induction resume values from the additional bypass block.
+ IRBuilder<> BypassBuilder(BypassBlock, BypassBlock->getFirstInsertionPt());
+ for (const auto &[IVPhi, II] : LVL.getInductionVars()) {
+ auto *Inc = cast<PHINode>(IVPhi->getIncomingValueForBlock(PH));
+ Value *V = createInductionAdditionalBypassValues(
+ IVPhi, II, BypassBuilder, ExpandedSCEVs, MainVectorTripCount,
+ LVL.getPrimaryInduction());
+ // TODO: Directly add as extra operand to the VPResumePHI recipe.
+ Inc->setIncomingValueForBlock(BypassBlock, V);
+ }
+}
+
bool LoopVectorizePass::processLoop(Loop *L) {
assert((EnableVPlanNativePath || L->isInnermost()) &&
"VPlan-native path is not enabled. Only process inner loops.");
@@ -10227,18 +10238,9 @@ bool LoopVectorizePass::processLoop(Loop *L) {
LVP.executePlan(EPI.EpilogueVF, EPI.EpilogueUF, BestEpiPlan, EpilogILV, DT,
true);
- // Fix induction resume values from the additional bypass block.
- BasicBlock *BypassBlock = EpilogILV.getAdditionalBypassBlock();
- IRBuilder<> BypassBuilder(BypassBlock, BypassBlock->getFirstInsertionPt());
- BasicBlock *PH = L->getLoopPreheader();
- for (const auto &[IVPhi, II] : LVL.getInductionVars()) {
- auto *Inc = cast<PHINode>(IVPhi->getIncomingValueForBlock(PH));
- Value *V = createInductionAdditionalBypassValues(
- IVPhi, II, BypassBuilder, ExpandedSCEVs, EPI.VectorTripCount,
- LVL.getPrimaryInduction());
- // TODO: Directly add as extra operand to the VPResumePHI recipe.
- Inc->setIncomingValueForBlock(BypassBlock, V);
- }
+ fixScalarResumeValuesFromBypass(EpilogILV.getAdditionalBypassBlock(), L,
+ BestEpiPlan, LVL, ExpandedSCEVs,
+ EPI.VectorTripCount);
++LoopsEpilogueVectorized;
if (!Checks.hasChecks())
More information about the llvm-commits
mailing list