[llvm] 9d45077 - [VPlan] Iterate over VPlans to get VFs to compute cost for (NFCI).
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 25 02:48:01 PDT 2024
Author: Florian Hahn
Date: 2024-06-25T10:47:51+01:00
New Revision: 9d45077df92a64ff8db5b249ac2f2d45e82ab73c
URL: https://github.com/llvm/llvm-project/commit/9d45077df92a64ff8db5b249ac2f2d45e82ab73c
DIFF: https://github.com/llvm/llvm-project/commit/9d45077df92a64ff8db5b249ac2f2d45e82ab73c.diff
LOG: [VPlan] Iterate over VPlans to get VFs to compute cost for (NFCI).
Instead for iterating over all VFs when computing costs, simply iterate
over the VFs available in the created VPlans.
Split off from https://github.com/llvm/llvm-project/pull/92555.
This also prepares for moving the check if any vector instructions will
be generated to be based on VPlan, to unblock recommitting
https://github.com/llvm/llvm-project/pull/92555.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index c03c278fcebe7..93f6d1d82e244 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -262,16 +262,6 @@ struct VectorizationFactor {
}
};
-/// ElementCountComparator creates a total ordering for ElementCount
-/// for the purposes of using it in a set structure.
-struct ElementCountComparator {
- bool operator()(const ElementCount &LHS, const ElementCount &RHS) const {
- return std::make_tuple(LHS.isScalable(), LHS.getKnownMinValue()) <
- std::make_tuple(RHS.isScalable(), RHS.getKnownMinValue());
- }
-};
-using ElementCountSet = SmallSet<ElementCount, 16, ElementCountComparator>;
-
/// A class that represents two vectorization factors (initialized with 0 by
/// default). One for fixed-width vectorization and one for scalable
/// vectorization. This can be used by the vectorizer to choose from a range of
@@ -442,10 +432,9 @@ class LoopVectorizationPlanner {
VPRecipeBuilder &RecipeBuilder,
ElementCount MinVF);
- /// \return The most profitable vectorization factor and the cost of that VF.
- /// This method checks every VF in \p CandidateVFs.
- VectorizationFactor
- selectVectorizationFactor(const ElementCountSet &CandidateVFs);
+ /// \return The most profitable vectorization factor for the available VPlans
+ /// and the cost of that VF.
+ VectorizationFactor selectVectorizationFactor();
/// Returns true if the per-lane cost of VectorizationFactor A is lower than
/// that of B.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 771fb247f201f..7835836d21ef1 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4833,8 +4833,10 @@ static void emitInvalidCostRemarks(SmallVector<InstructionVFPair> InvalidCosts,
sort(InvalidCosts, [&Numbering](InstructionVFPair &A, InstructionVFPair &B) {
if (Numbering[A.first] != Numbering[B.first])
return Numbering[A.first] < Numbering[B.first];
- ElementCountComparator ECC;
- return ECC(A.second, B.second);
+ const auto &LHS = A.second;
+ const auto &RHS = B.second;
+ return std::make_tuple(LHS.isScalable(), LHS.getKnownMinValue()) <
+ std::make_tuple(RHS.isScalable(), RHS.getKnownMinValue());
});
// For a list of ordered instruction-vf pairs:
@@ -4877,13 +4879,15 @@ static void emitInvalidCostRemarks(SmallVector<InstructionVFPair> InvalidCosts,
} while (!Tail.empty());
}
-VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor(
- const ElementCountSet &VFCandidates) {
+VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
InstructionCost ExpectedCost =
CM.expectedCost(ElementCount::getFixed(1)).first;
LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ExpectedCost << ".\n");
assert(ExpectedCost.isValid() && "Unexpected invalid cost for scalar loop");
- assert(VFCandidates.count(ElementCount::getFixed(1)) &&
+ assert(any_of(VPlans,
+ [](std::unique_ptr<VPlan> &P) {
+ return P->hasVF(ElementCount::getFixed(1));
+ }) &&
"Expected Scalar VF to be a candidate");
const VectorizationFactor ScalarCost(ElementCount::getFixed(1), ExpectedCost,
@@ -4891,7 +4895,8 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor(
VectorizationFactor ChosenFactor = ScalarCost;
bool ForceVectorization = Hints.getForce() == LoopVectorizeHints::FK_Enabled;
- if (ForceVectorization && VFCandidates.size() > 1) {
+ if (ForceVectorization &&
+ (VPlans.size() > 1 || !VPlans[0]->hasScalarVFOnly())) {
// Ignore scalar width, because the user explicitly wants vectorization.
// Initialize cost to max so that VF = 2 is, at least, chosen during cost
// evaluation.
@@ -4899,43 +4904,46 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor(
}
SmallVector<InstructionVFPair> InvalidCosts;
- for (const auto &i : VFCandidates) {
- // The cost for scalar VF=1 is already calculated, so ignore it.
- if (i.isScalar())
- continue;
+ for (auto &P : VPlans) {
+ for (ElementCount VF : P->vectorFactors()) {
+ // The cost for scalar VF=1 is already calculated, so ignore it.
+ if (VF.isScalar())
+ continue;
- LoopVectorizationCostModel::VectorizationCostTy C =
- CM.expectedCost(i, &InvalidCosts);
- VectorizationFactor Candidate(i, C.first, ScalarCost.ScalarCost);
+ LoopVectorizationCostModel::VectorizationCostTy C =
+ CM.expectedCost(VF, &InvalidCosts);
+ VectorizationFactor Candidate(VF, C.first, ScalarCost.ScalarCost);
#ifndef NDEBUG
- unsigned AssumedMinimumVscale =
- getVScaleForTuning(OrigLoop, TTI).value_or(1);
- unsigned Width =
- Candidate.Width.isScalable()
- ? Candidate.Width.getKnownMinValue() * AssumedMinimumVscale
- : Candidate.Width.getFixedValue();
- LLVM_DEBUG(dbgs() << "LV: Vector loop of width " << i
- << " costs: " << (Candidate.Cost / Width));
- if (i.isScalable())
- LLVM_DEBUG(dbgs() << " (assuming a minimum vscale of "
- << AssumedMinimumVscale << ")");
- LLVM_DEBUG(dbgs() << ".\n");
+ unsigned AssumedMinimumVscale =
+ getVScaleForTuning(OrigLoop, TTI).value_or(1);
+ unsigned Width =
+ Candidate.Width.isScalable()
+ ? Candidate.Width.getKnownMinValue() * AssumedMinimumVscale
+ : Candidate.Width.getFixedValue();
+ LLVM_DEBUG(dbgs() << "LV: Vector loop of width " << VF
+ << " costs: " << (Candidate.Cost / Width));
+ if (VF.isScalable())
+ LLVM_DEBUG(dbgs() << " (assuming a minimum vscale of "
+ << AssumedMinimumVscale << ")");
+ LLVM_DEBUG(dbgs() << ".\n");
#endif
- if (!C.second && !ForceVectorization) {
- LLVM_DEBUG(
- dbgs() << "LV: Not considering vector loop of width " << i
- << " because it will not generate any vector instructions.\n");
- continue;
- }
+ if (!C.second && !ForceVectorization) {
+ LLVM_DEBUG(
+ dbgs()
+ << "LV: Not considering vector loop of width " << VF
+ << " because it will not generate any vector instructions.\n");
+ continue;
+ }
- // If profitable add it to ProfitableVF list.
- if (isMoreProfitable(Candidate, ScalarCost))
- ProfitableVFs.push_back(Candidate);
+ // If profitable add it to ProfitableVF list.
+ if (isMoreProfitable(Candidate, ScalarCost))
+ ProfitableVFs.push_back(Candidate);
- if (isMoreProfitable(Candidate, ChosenFactor))
- ChosenFactor = Candidate;
+ if (isMoreProfitable(Candidate, ChosenFactor))
+ ChosenFactor = Candidate;
+ }
}
emitInvalidCostRemarks(InvalidCosts, ORE, OrigLoop);
@@ -7270,14 +7278,14 @@ LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
"InvalidCost", ORE, OrigLoop);
}
- // Populate the set of Vectorization Factor Candidates.
- ElementCountSet VFCandidates;
+ // Collect the Vectorization Factor Candidates.
+ SmallVector<ElementCount> VFCandidates;
for (auto VF = ElementCount::getFixed(1);
ElementCount::isKnownLE(VF, MaxFactors.FixedVF); VF *= 2)
- VFCandidates.insert(VF);
+ VFCandidates.push_back(VF);
for (auto VF = ElementCount::getScalable(1);
ElementCount::isKnownLE(VF, MaxFactors.ScalableVF); VF *= 2)
- VFCandidates.insert(VF);
+ VFCandidates.push_back(VF);
CM.collectInLoopReductions();
for (const auto &VF : VFCandidates) {
@@ -7299,7 +7307,7 @@ LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
return VectorizationFactor::Disabled();
// Select the optimal vectorization factor.
- VectorizationFactor VF = selectVectorizationFactor(VFCandidates);
+ VectorizationFactor VF = selectVectorizationFactor();
assert((VF.Width.isScalar() || VF.ScalarCost > 0) && "when vectorizing, the scalar cost must be non-zero.");
if (!hasPlanWithVF(VF.Width)) {
LLVM_DEBUG(dbgs() << "LV: No VPlan could be built for " << VF.Width
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 8916bab2aaf06..db10c7a240c7e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -3255,6 +3255,12 @@ class VPlan {
return any_of(VFs, [](ElementCount VF) { return VF.isScalable(); });
}
+ /// Returns an iterator range over all VFs of the plan.
+ iterator_range<SmallSetVector<ElementCount, 2>::iterator>
+ vectorFactors() const {
+ return {VFs.begin(), VFs.end()};
+ }
+
bool hasScalarVFOnly() const { return VFs.size() == 1 && VFs[0].isScalar(); }
bool hasUF(unsigned UF) const { return UFs.empty() || UFs.contains(UF); }
More information about the llvm-commits
mailing list