[llvm] 5b4b3e0 - [VPlan] Move widening check for non-memory/non-calls to function (NFC).
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 15 03:50:04 PDT 2020
Author: Florian Hahn
Date: 2020-04-15T11:48:37+01:00
New Revision: 5b4b3e0b6e930cf993b9aaadbe709bfb96573598
URL: https://github.com/llvm/llvm-project/commit/5b4b3e0b6e930cf993b9aaadbe709bfb96573598
DIFF: https://github.com/llvm/llvm-project/commit/5b4b3e0b6e930cf993b9aaadbe709bfb96573598.diff
LOG: [VPlan] Move widening check for non-memory/non-calls to function (NFC).
After introducing VPWidenSelectRecipe, the duplicated logic can be
shared.
Reviewers: gilr, rengolin, Ayal, hsaito
Reviewed By: Ayal
Differential Revision: https://reviews.llvm.org/D77973
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1ef5e059644d..0e5079533509 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6931,22 +6931,24 @@ VPRecipeBuilder::tryToWidenCall(Instruction *I, VFRange &Range, VPlan &Plan) {
return new VPWidenCallRecipe(*CI, VPValues);
}
-VPWidenSelectRecipe *VPRecipeBuilder::tryToWidenSelect(Instruction *I,
- VFRange &Range) {
- auto *SI = dyn_cast<SelectInst>(I);
- if (!SI)
- return nullptr;
-
- // SI should be widened, unless it is scalar after vectorization,
+bool VPRecipeBuilder::shouldWiden(Instruction *I, VFRange &Range) const {
+ assert(!isa<PHINode>(I) && !isa<LoadInst>(I) && !isa<StoreInst>(I) &&
+ "Instruction should have been handled earlier");
+ // Instruction should be widened, unless it is scalar after vectorization,
// scalarization is profitable or it is predicated.
- auto willWiden = [this, SI](unsigned VF) -> bool {
- return !CM.isScalarAfterVectorization(SI, VF) &&
- !CM.isProfitableToScalarize(SI, VF) &&
- !CM.isScalarWithPredication(SI, VF);
+ auto WillScalarize = [this, I](unsigned VF) -> bool {
+ return CM.isScalarAfterVectorization(I, VF) ||
+ CM.isProfitableToScalarize(I, VF) ||
+ CM.isScalarWithPredication(I, VF);
};
- if (!LoopVectorizationPlanner::getDecisionAndClampRange(willWiden, Range))
- return nullptr;
+ return !LoopVectorizationPlanner::getDecisionAndClampRange(WillScalarize,
+ Range);
+}
+VPWidenSelectRecipe *VPRecipeBuilder::tryToWidenSelect(Instruction *I) {
+ auto *SI = dyn_cast<SelectInst>(I);
+ if (!SI)
+ return nullptr;
auto *SE = PSE.getSE();
bool InvariantCond =
SE->isLoopInvariant(PSE.getSCEV(SI->getOperand(0)), OrigLoop);
@@ -6954,14 +6956,7 @@ VPWidenSelectRecipe *VPRecipeBuilder::tryToWidenSelect(Instruction *I,
return new VPWidenSelectRecipe(*SI, InvariantCond);
}
-VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I, VFRange &Range) {
- bool IsPredicated = LoopVectorizationPlanner::getDecisionAndClampRange(
- [&](unsigned VF) { return CM.isScalarWithPredication(I, VF); }, Range);
-
- if (IsPredicated)
- return nullptr;
-
- assert(!isa<PHINode>(I) && "PHIs should have been handled earlier");
+VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I, VPlan &Plan) {
auto IsVectorizableOpcode = [](unsigned Opcode) {
switch (Opcode) {
case Instruction::Add:
@@ -7007,14 +7002,6 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I, VFRange &Range) {
if (!IsVectorizableOpcode(I->getOpcode()))
return nullptr;
- auto willWiden = [&](unsigned VF) -> bool {
- return !CM.isScalarAfterVectorization(I, VF) &&
- !CM.isProfitableToScalarize(I, VF);
- };
-
- if (!LoopVectorizationPlanner::getDecisionAndClampRange(willWiden, Range))
- return nullptr;
-
// Success: widen this instruction.
return new VPWidenRecipe(*I);
}
@@ -7095,7 +7082,6 @@ bool VPRecipeBuilder::tryToCreateRecipe(Instruction *Instr, VFRange &Range,
// operations, inductions and Phi nodes.
if ((Recipe = tryToWidenCall(Instr, Range, *Plan)) ||
(Recipe = tryToWidenMemory(Instr, Range, Plan)) ||
- (Recipe = tryToWidenSelect(Instr, Range)) ||
(Recipe = tryToOptimizeInduction(Instr, Range)) ||
(Recipe = tryToBlend(Instr, Plan)) ||
(isa<PHINode>(Instr) &&
@@ -7105,24 +7091,19 @@ bool VPRecipeBuilder::tryToCreateRecipe(Instruction *Instr, VFRange &Range,
return true;
}
- // Handle GEP widening.
- if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Instr)) {
- auto Scalarize = [&](unsigned VF) {
- return CM.isScalarWithPredication(Instr, VF) ||
- CM.isScalarAfterVectorization(Instr, VF) ||
- CM.isProfitableToScalarize(Instr, VF);
- };
- if (LoopVectorizationPlanner::getDecisionAndClampRange(Scalarize, Range))
- return false;
- VPWidenGEPRecipe *Recipe = new VPWidenGEPRecipe(GEP, OrigLoop);
- setRecipe(Instr, Recipe);
- VPBB->appendRecipe(Recipe);
- return true;
- }
+ // Calls and memory instructions are widened by the specialized recipes above,
+ // or scalarized.
+ if (isa<CallInst>(Instr) || isa<LoadInst>(Instr) || isa<StoreInst>(Instr))
+ return false;
+
+ if (!shouldWiden(Instr, Range))
+ return false;
- // Check if Instr is to be widened by a general VPWidenRecipe, after
- // having first checked for specific widening recipes.
- if ((Recipe = tryToWiden(Instr, Range))) {
+ if ((Recipe = tryToWidenSelect(Instr)) ||
+ (isa<GetElementPtrInst>(Instr) &&
+ (Recipe =
+ new VPWidenGEPRecipe(cast<GetElementPtrInst>(Instr), OrigLoop))) ||
+ (Recipe = tryToWiden(Instr, *Plan))) {
setRecipe(Instr, Recipe);
VPBB->appendRecipe(Recipe);
return true;
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index a12c3f33ab53..46518020ea9b 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -64,6 +64,11 @@ class VPRecipeBuilder {
Ingredient2Recipe[I] = R;
}
+ /// Check if \p I can be widened at the start of \p Range and possibly
+ /// decrease the range such that the returned value holds for the entire \p
+ /// Range. The function should not be called for memory instructions or calls.
+ bool shouldWiden(Instruction *I, VFRange &Range) const;
+
public:
/// A helper function that computes the predicate of the block BB, assuming
/// that the header block of the loop is set to True. It returns the *entry*
@@ -114,14 +119,15 @@ class VPRecipeBuilder {
/// ensure same decision from \p Range.Start to \p Range.End.
VPWidenCallRecipe *tryToWidenCall(Instruction *I, VFRange &Range,
VPlan &Plan);
-
- VPWidenSelectRecipe *tryToWidenSelect(Instruction *I, VFRange &Range);
-
- /// Check if \p I can be widened within the given VF \p Range. If \p I can be
- /// widened for \p Range.Start, build a new VPWidenRecipe and return it.
- /// Range.End may be decreased to ensure same decision from \p Range.Start to
- /// \p Range.End.
- VPWidenRecipe *tryToWiden(Instruction *I, VFRange &Range);
+ /// Check if \p I is a SelectInst and return a VPWidenSelectRecipe if it is.
+ /// The function should only be called if the cost-model indicates that
+ /// widening should be performed.
+ VPWidenSelectRecipe *tryToWidenSelect(Instruction *I);
+
+ /// Check if \p I has an opcode that can be widened and return a VPWidenRecipe
+ /// if it can. The function should only be called if the cost-model indicates
+ /// that widening should be performed.
+ VPWidenRecipe *tryToWiden(Instruction *I, VPlan &Plan);
/// Create a replicating region for instruction \p I that requires
/// predication. \p PredRecipe is a VPReplicateRecipe holding \p I.
More information about the llvm-commits
mailing list