[llvm] fc93689 - [LV] Move isCandidateForEpilogueVec to LoopVectorizationPlanner.cpp (NFC). (#195268)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 24 03:08:43 PDT 2026
Author: Florian Hahn
Date: 2026-08-24T10:08:38Z
New Revision: fc936894d95c0608c598cad3147d83ae4d43a237
URL: https://github.com/llvm/llvm-project/commit/fc936894d95c0608c598cad3147d83ae4d43a237
DIFF: https://github.com/llvm/llvm-project/commit/fc936894d95c0608c598cad3147d83ae4d43a237.diff
LOG: [LV] Move isCandidateForEpilogueVec to LoopVectorizationPlanner.cpp (NFC). (#195268)
isCandidateForEpilogueVectorization (together with its file-static
helper hasUnsupportedHeaderPhiRecipe) does not depend on anything in
LoopVectorize.cpp, move it to LoopVectorizationPlanner.cpp.
PR: https://github.com/llvm/llvm-project/pull/195268
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
index cbe2f63f96005..c464c7894ae5c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.cpp
@@ -13,6 +13,7 @@
//===----------------------------------------------------------------------===//
#include "LoopVectorizationPlanner.h"
+#include "VPlanUtils.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ScalarEvolution.h"
@@ -856,3 +857,61 @@ VFSelectionContext::computeVPlanOuterloopVF(ElementCount UserVF) {
<< "VF " << VF << " to build VPlans.\n");
return FixedScalableVFPair(VF);
}
+
+/// \returns true if the VPlan contains header phi recipes that are not
+/// currently supported for epilogue vectorization.
+static bool hasUnsupportedHeaderPhiRecipe(VPlan &Plan) {
+ return any_of(
+ Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
+ [](VPRecipeBase &R) {
+ switch (R.getVPRecipeID()) {
+ case VPRecipeBase::VPFirstOrderRecurrencePHISC:
+ // TODO: Add support for fixed-order recurrences.
+ return true;
+ case VPRecipeBase::VPWidenIntOrFpInductionSC:
+ return !cast<VPWidenIntOrFpInductionRecipe>(&R)->getPHINode();
+ case VPRecipeBase::VPReductionPHISC: {
+ auto *RedPhi = cast<VPReductionPHIRecipe>(&R);
+ // TODO: Support FMinNum/FMaxNum, FindLast reductions, and reductions
+ // without underlying values.
+ RecurKind Kind = RedPhi->getRecurrenceKind();
+ if (RecurrenceDescriptor::isFPMinMaxNumRecurrenceKind(Kind) ||
+ RecurrenceDescriptor::isFindLastRecurrenceKind(Kind) ||
+ !RedPhi->getUnderlyingValue())
+ return true;
+ // TODO: Add support for FindIV reductions with sunk expressions: the
+ // resume value from the main loop is in expression domain (e.g.,
+ // mul(ReducedIV, 3)), but the epilogue tracks raw IV values. A sunk
+ // expression is identified by a non-VPInstruction user of
+ // ComputeReductionResult.
+ if (RecurrenceDescriptor::isFindIVRecurrenceKind(Kind)) {
+ auto *RdxResult = vputils::findComputeReductionResult(RedPhi);
+ assert(RdxResult &&
+ "FindIV reduction must have ComputeReductionResult");
+ return any_of(RdxResult->users(),
+ std::not_fn(IsaPred<VPInstruction>));
+ }
+ return false;
+ }
+ default:
+ return false;
+ };
+ });
+}
+
+bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
+ VPlan &MainPlan) const {
+ // Bail out if the plan contains header phi recipes not yet supported
+ // for epilogue vectorization.
+ if (hasUnsupportedHeaderPhiRecipe(MainPlan))
+ return false;
+
+ // Epilogue vectorization code has not been auditted to ensure it handles
+ // non-latch exits properly. It may be fine, but it needs auditted and
+ // tested.
+ // TODO: Add support for loops with an early exit.
+ if (OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch())
+ return false;
+
+ return true;
+}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index f578a87c9d9cc..9ca869f5ebe88 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -755,6 +755,10 @@ class VFSelectionContext {
/// \return The vectorization hints for the loop being analyzed.
const LoopVectorizeHints &getHints() const { return *Hints; }
+ /// Returns true if epilogue vectorization is considered profitable for a
+ /// main loop with vectorization factor \p VF and interleave count \p IC.
+ bool isEpilogueVectorizationProfitable(ElementCount VF, unsigned IC) const;
+
/// \return True if register pressure should be considered for the given VF.
bool shouldConsiderRegPressureForVF(ElementCount VF) const;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index bae9fb9db902f..5e8a43eb871c5 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1273,14 +1273,6 @@ class LoopVectorizationCostModel {
/// the factor width.
InstructionCost expectedCost(ElementCount VF);
- /// Returns true if epilogue vectorization is considered profitable, and
- /// false otherwise.
- /// \p VF is the vectorization factor chosen for the original loop.
- /// \p Multiplier is an aditional scaling factor applied to VF before
- /// comparing to EpilogueVectorizationMinVF.
- bool isEpilogueVectorizationProfitable(const ElementCount VF,
- const unsigned IC) const;
-
/// Returns the execution time cost of an instruction for a given vector
/// width. Vector width of one means scalar.
InstructionCost getInstructionCost(Instruction *I, ElementCount VF);
@@ -3359,66 +3351,7 @@ static bool hasFindLastReductionPhi(VPlan &Plan) {
RedPhi->getRecurrenceKind());
});
}
-
-/// Returns true if the VPlan contains header phi recipes that are not currently
-/// supported for epilogue vectorization.
-static bool hasUnsupportedHeaderPhiRecipe(VPlan &Plan) {
- return any_of(
- Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
- [](VPRecipeBase &R) {
- switch (R.getVPRecipeID()) {
- case VPRecipeBase::VPFirstOrderRecurrencePHISC:
- // TODO: Add support for fixed-order recurrences.
- return true;
- case VPRecipeBase::VPWidenIntOrFpInductionSC:
- return !cast<VPWidenIntOrFpInductionRecipe>(&R)->getPHINode();
- case VPRecipeBase::VPReductionPHISC: {
- auto *RedPhi = cast<VPReductionPHIRecipe>(&R);
- // TODO: Support FMinNum/FMaxNum, FindLast reductions, and reductions
- // without underlying values.
- RecurKind Kind = RedPhi->getRecurrenceKind();
- if (RecurrenceDescriptor::isFPMinMaxNumRecurrenceKind(Kind) ||
- RecurrenceDescriptor::isFindLastRecurrenceKind(Kind) ||
- !RedPhi->getUnderlyingValue())
- return true;
- // TODO: Add support for FindIV reductions with sunk expressions: the
- // resume value from the main loop is in expression domain (e.g.,
- // mul(ReducedIV, 3)), but the epilogue tracks raw IV values. A sunk
- // expression is identified by a non-VPInstruction user of
- // ComputeReductionResult.
- if (RecurrenceDescriptor::isFindIVRecurrenceKind(Kind)) {
- auto *RdxResult = vputils::findComputeReductionResult(RedPhi);
- assert(RdxResult &&
- "FindIV reduction must have ComputeReductionResult");
- return any_of(RdxResult->users(),
- std::not_fn(IsaPred<VPInstruction>));
- }
- return false;
- }
- default:
- return false;
- };
- });
-}
-
-bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
- VPlan &MainPlan) const {
- // Bail out if the plan contains header phi recipes not yet supported
- // for epilogue vectorization.
- if (hasUnsupportedHeaderPhiRecipe(MainPlan))
- return false;
-
- // Epilogue vectorization code has not been auditted to ensure it handles
- // non-latch exits properly. It may be fine, but it needs auditted and
- // tested.
- // TODO: Add support for loops with an early exit.
- if (OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch())
- return false;
-
- return true;
-}
-
-bool LoopVectorizationCostModel::isEpilogueVectorizationProfitable(
+bool VFSelectionContext::isEpilogueVectorizationProfitable(
const ElementCount VF, const unsigned IC) const {
// FIXME: We need a much better cost-model to take
diff erent parameters such
// as register pressure, code size increase and cost of extra branches into
@@ -3432,8 +3365,7 @@ bool LoopVectorizationCostModel::isEpilogueVectorizationProfitable(
unsigned MinVFThreshold = EpilogueVectorizationMinVF.getNumOccurrences() > 0
? EpilogueVectorizationMinVF
: TTI.getEpilogueVectorizationMinVF();
- return estimateElementCount(VF * IC, Config.getVScaleForTuning()) >=
- MinVFThreshold;
+ return estimateElementCount(VF * IC, getVScaleForTuning()) >= MinVFThreshold;
}
std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
@@ -3495,7 +3427,7 @@ std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
return nullptr;
}
- if (!CM.isEpilogueVectorizationProfitable(MainLoopVF, IC)) {
+ if (!Config.isEpilogueVectorizationProfitable(MainLoopVF, IC)) {
LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is not profitable for "
"this loop\n");
return nullptr;
More information about the llvm-commits
mailing list