[llvm] f47573f - [LoopVectorizer] NFC: Propagate ElementCount to more interfaces.
Sander de Smalen via llvm-commits
llvm-commits at lists.llvm.org
Tue Nov 10 03:12:03 PST 2020
Author: Sander de Smalen
Date: 2020-11-10T11:11:02Z
New Revision: f47573f9bfe55137c10d88c8e30c2974ec375062
URL: https://github.com/llvm/llvm-project/commit/f47573f9bfe55137c10d88c8e30c2974ec375062
DIFF: https://github.com/llvm/llvm-project/commit/f47573f9bfe55137c10d88c8e30c2974ec375062.diff
LOG: [LoopVectorizer] NFC: Propagate ElementCount to more interfaces.
Interfaces changed to take `ElementCount` as parameters:
* LoopVectorizationPlanner::buildVPlans
* LoopVectorizationPlanner::buildVPlansWithVPRecipes
* LoopVectorizationCostModel::selectVectorizationFactor
This patch is NFC for fixed-width vectors.
Reviewed By: dmgreen, ctetreau
Differential Revision: https://reviews.llvm.org/D90879
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 8c3dff69e072..b3b744947c1f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -281,7 +281,7 @@ class LoopVectorizationPlanner {
/// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
/// according to the information gathered by Legal when it checked if it is
/// legal to vectorize the loop.
- void buildVPlans(unsigned MinVF, unsigned MaxVF);
+ void buildVPlans(ElementCount MinVF, ElementCount MaxVF);
private:
/// Build a VPlan according to the information gathered by Legal. \return a
@@ -299,7 +299,7 @@ class LoopVectorizationPlanner {
/// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
/// according to the information gathered by Legal when it checked if it is
/// legal to vectorize the loop. This method creates VPlans using VPRecipes.
- void buildVPlansWithVPRecipes(unsigned MinVF, unsigned MaxVF);
+ void buildVPlansWithVPRecipes(ElementCount MinVF, ElementCount MaxVF);
/// Adjust the recipes for any inloop reductions. The chain of instructions
/// leading from the loop exit instr to the phi need to be converted to
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e47cde990eca..2e195669af62 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1075,7 +1075,7 @@ class LoopVectorizationCostModel {
/// This method checks every power of two up to MaxVF. If UserVF is not ZERO
/// then this vectorization factor will be selected if vectorization is
/// possible.
- VectorizationFactor selectVectorizationFactor(unsigned MaxVF);
+ VectorizationFactor selectVectorizationFactor(ElementCount MaxVF);
/// Setup cost-based decisions for user vectorization factor.
void selectUserVectorizationFactor(ElementCount UserVF) {
@@ -5400,21 +5400,23 @@ LoopVectorizationCostModel::computeFeasibleMaxVF(unsigned ConstTripCount) {
}
VectorizationFactor
-LoopVectorizationCostModel::selectVectorizationFactor(unsigned MaxVF) {
+LoopVectorizationCostModel::selectVectorizationFactor(ElementCount MaxVF) {
+ assert(!MaxVF.isScalable() && "scalable vectors not yet supported");
+
float Cost = expectedCost(ElementCount::getFixed(1)).first;
const float ScalarCost = Cost;
unsigned Width = 1;
LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << (int)ScalarCost << ".\n");
bool ForceVectorization = Hints->getForce() == LoopVectorizeHints::FK_Enabled;
- if (ForceVectorization && MaxVF > 1) {
+ if (ForceVectorization && MaxVF.isVector()) {
// Ignore scalar width, because the user explicitly wants vectorization.
// Initialize cost to max so that VF = 2 is, at least, chosen during cost
// evaluation.
Cost = std::numeric_limits<float>::max();
}
- for (unsigned i = 2; i <= MaxVF; i *= 2) {
+ for (unsigned i = 2; i <= MaxVF.getFixedValue(); i *= 2) {
// Notice that the vector loop needs to be executed less times, so
// we need to divide the cost of the vector loops by the width of
// the vector elements.
@@ -6963,7 +6965,7 @@ LoopVectorizationPlanner::planInVPlanNativePath(ElementCount UserVF) {
"VF needs to be a power of two");
LLVM_DEBUG(dbgs() << "LV: Using " << (!UserVF.isZero() ? "user " : "")
<< "VF " << VF << " to build VPlans.\n");
- buildVPlans(VF.getKnownMinValue(), VF.getKnownMinValue());
+ buildVPlans(VF, VF);
// For VPlan build stress testing, we bail out after VPlan construction.
if (VPlanBuildStressTest)
@@ -7009,30 +7011,30 @@ LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
// profitable to scalarize.
CM.selectUserVectorizationFactor(UserVF);
CM.collectInLoopReductions();
- buildVPlansWithVPRecipes(UserVF.getKnownMinValue(),
- UserVF.getKnownMinValue());
+ buildVPlansWithVPRecipes(UserVF, UserVF);
LLVM_DEBUG(printPlans(dbgs()));
return {{UserVF, 0}};
}
- unsigned MaxVF = MaybeMaxVF.getValue();
- assert(MaxVF != 0 && "MaxVF is zero.");
+ ElementCount MaxVF = ElementCount::getFixed(MaybeMaxVF.getValue());
+ assert(MaxVF.isNonZero() && "MaxVF is zero.");
- for (unsigned VF = 1; VF <= MaxVF; VF *= 2) {
+ for (ElementCount VF = ElementCount::getFixed(1);
+ ElementCount::isKnownLE(VF, MaxVF); VF *= 2) {
// Collect Uniform and Scalar instructions after vectorization with VF.
- CM.collectUniformsAndScalars(ElementCount::getFixed(VF));
+ CM.collectUniformsAndScalars(VF);
// Collect the instructions (and their associated costs) that will be more
// profitable to scalarize.
- if (VF > 1)
- CM.collectInstsToScalarize(ElementCount::getFixed(VF));
+ if (VF.isVector())
+ CM.collectInstsToScalarize(VF);
}
CM.collectInLoopReductions();
- buildVPlansWithVPRecipes(1, MaxVF);
+ buildVPlansWithVPRecipes(ElementCount::getFixed(1), MaxVF);
LLVM_DEBUG(printPlans(dbgs()));
- if (MaxVF == 1)
+ if (MaxVF.isScalar())
return VectorizationFactor::Disabled();
// Select the optimal vectorization factor.
@@ -7202,10 +7204,10 @@ bool LoopVectorizationPlanner::getDecisionAndClampRange(
/// of VF's starting at a given VF and extending it as much as possible. Each
/// vectorization decision can potentially shorten this sub-range during
/// buildVPlan().
-void LoopVectorizationPlanner::buildVPlans(unsigned MinVF, unsigned MaxVF) {
- auto MaxVFPlusOne = ElementCount::getFixed(MaxVF).getWithIncrement(1);
- for (ElementCount VF = ElementCount::getFixed(MinVF);
- ElementCount::isKnownLT(VF, MaxVFPlusOne);) {
+void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF,
+ ElementCount MaxVF) {
+ auto MaxVFPlusOne = MaxVF.getWithIncrement(1);
+ for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFPlusOne);) {
VFRange SubRange = {VF, MaxVFPlusOne};
VPlans.push_back(buildVPlan(SubRange));
VF = SubRange.End;
@@ -7612,8 +7614,8 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
return tryToWiden(Instr, *Plan);
}
-void LoopVectorizationPlanner::buildVPlansWithVPRecipes(unsigned MinVF,
- unsigned MaxVF) {
+void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
+ ElementCount MaxVF) {
assert(OrigLoop->isInnermost() && "Inner loop expected.");
// Collect conditions feeding internal conditional branches; they need to be
@@ -7663,9 +7665,8 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(unsigned MinVF,
for (Instruction *I : DeadInstructions)
SinkAfter.erase(I);
- auto MaxVFPlusOne = ElementCount::getFixed(MaxVF).getWithIncrement(1);
- for (ElementCount VF = ElementCount::getFixed(MinVF);
- ElementCount::isKnownLT(VF, MaxVFPlusOne);) {
+ auto MaxVFPlusOne = MaxVF.getWithIncrement(1);
+ for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFPlusOne);) {
VFRange SubRange = {VF, MaxVFPlusOne};
VPlans.push_back(buildVPlanWithVPRecipes(SubRange, NeedDef,
DeadInstructions, SinkAfter));
More information about the llvm-commits
mailing list