[llvm] e0c479c - [VPlan] Switch VPWidenRecipe to be a VPValue
David Green via llvm-commits
llvm-commits at lists.llvm.org
Wed Nov 25 00:26:38 PST 2020
Author: David Green
Date: 2020-11-25T08:25:06Z
New Revision: e0c479cd0e03279784925ece209ff53bdbb86cf8
URL: https://github.com/llvm/llvm-project/commit/e0c479cd0e03279784925ece209ff53bdbb86cf8
DIFF: https://github.com/llvm/llvm-project/commit/e0c479cd0e03279784925ece209ff53bdbb86cf8.diff
LOG: [VPlan] Switch VPWidenRecipe to be a VPValue
Similar to other patches, this makes VPWidenRecipe a VPValue. Because of
the way it interacts with the reduction code it also slightly alters the
way that VPValues are registered, removing the up front NeedDef and
using getOrAddVPValue to create them on-demand if needed instead.
Differential Revision: https://reviews.llvm.org/D88447
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanValue.h
llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
llvm/test/Transforms/LoopVectorize/vplan-printing.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index b3b744947c1f..ec88bebe684d 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -292,8 +292,7 @@ class LoopVectorizationPlanner {
/// Build a VPlan using VPRecipes according to the information gather by
/// Legal. This method is only used for the legacy inner loop vectorizer.
VPlanPtr buildVPlanWithVPRecipes(
- VFRange &Range, SmallPtrSetImpl<Value *> &NeedDef,
- SmallPtrSetImpl<Instruction *> &DeadInstructions,
+ VFRange &Range, SmallPtrSetImpl<Instruction *> &DeadInstructions,
const DenseMap<Instruction *, Instruction *> &SinkAfter);
/// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 35af7a445eef..97c9011d8086 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -461,7 +461,7 @@ class InnerLoopVectorizer {
BasicBlock *createVectorizedLoopSkeleton();
/// Widen a single instruction within the innermost loop.
- void widenInstruction(Instruction &I, VPUser &Operands,
+ void widenInstruction(Instruction &I, VPValue *Def, VPUser &Operands,
VPTransformState &State);
/// Widen a single call instruction within the innermost loop.
@@ -4512,7 +4512,8 @@ static bool mayDivideByZero(Instruction &I) {
return !CInt || CInt->isZero();
}
-void InnerLoopVectorizer::widenInstruction(Instruction &I, VPUser &User,
+void InnerLoopVectorizer::widenInstruction(Instruction &I, VPValue *Def,
+ VPUser &User,
VPTransformState &State) {
assert(!VF.isScalable() && "scalable vectors not yet supported.");
switch (I.getOpcode()) {
@@ -4555,7 +4556,7 @@ void InnerLoopVectorizer::widenInstruction(Instruction &I, VPUser &User,
VecOp->copyIRFlags(&I);
// Use this vector value for all users of the original instruction.
- VectorLoopValueMap.setVectorValue(&I, Part, V);
+ State.set(Def, &I, V, Part);
addMetadata(V, &I);
}
@@ -4579,7 +4580,7 @@ void InnerLoopVectorizer::widenInstruction(Instruction &I, VPUser &User,
} else {
C = Builder.CreateICmp(Cmp->getPredicate(), A, B);
}
- VectorLoopValueMap.setVectorValue(&I, Part, C);
+ State.set(Def, &I, C, Part);
addMetadata(C, &I);
}
@@ -4609,7 +4610,7 @@ void InnerLoopVectorizer::widenInstruction(Instruction &I, VPUser &User,
for (unsigned Part = 0; Part < UF; ++Part) {
Value *A = State.get(User.getOperand(0), Part);
Value *Cast = Builder.CreateCast(CI->getOpcode(), A, DestTy);
- VectorLoopValueMap.setVectorValue(&I, Part, Cast);
+ State.set(Def, &I, Cast, Part);
addMetadata(Cast, &I);
}
break;
@@ -7262,7 +7263,7 @@ VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst,
if (!BI->isConditional() || BI->getSuccessor(0) == BI->getSuccessor(1))
return EdgeMaskCache[Edge] = SrcMask;
- VPValue *EdgeMask = Plan->getVPValue(BI->getCondition());
+ VPValue *EdgeMask = Plan->getOrAddVPValue(BI->getCondition());
assert(EdgeMask && "No Edge Mask found for condition");
if (BI->getSuccessor(0) != Dst)
@@ -7300,7 +7301,7 @@ VPValue *VPRecipeBuilder::createBlockInMask(BasicBlock *BB, VPlanPtr &Plan) {
// Start by constructing the desired canonical IV.
VPValue *IV = nullptr;
if (Legal->getPrimaryInduction())
- IV = Plan->getVPValue(Legal->getPrimaryInduction());
+ IV = Plan->getOrAddVPValue(Legal->getPrimaryInduction());
else {
auto IVRecipe = new VPWidenCanonicalIVRecipe();
Builder.getInsertBlock()->insert(IVRecipe, NewInsertionPoint);
@@ -7648,24 +7649,6 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
ElementCount MaxVF) {
assert(OrigLoop->isInnermost() && "Inner loop expected.");
- // Collect conditions feeding internal conditional branches; they need to be
- // represented in VPlan for it to model masking.
- SmallPtrSet<Value *, 1> NeedDef;
-
- auto *Latch = OrigLoop->getLoopLatch();
- for (BasicBlock *BB : OrigLoop->blocks()) {
- if (BB == Latch)
- continue;
- BranchInst *Branch = dyn_cast<BranchInst>(BB->getTerminator());
- if (Branch && Branch->isConditional())
- NeedDef.insert(Branch->getCondition());
- }
-
- // If the tail is to be folded by masking, the primary induction variable, if
- // exists needs to be represented in VPlan for it to model early-exit masking.
- if (CM.foldTailByMasking() && Legal->getPrimaryInduction())
- NeedDef.insert(Legal->getPrimaryInduction());
-
// Collect instructions from the original loop that will become trivially dead
// in the vectorized loop. We don't need to vectorize these instructions. For
// example, original induction update instructions can become dead because we
@@ -7690,15 +7673,14 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
auto MaxVFPlusOne = MaxVF.getWithIncrement(1);
for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFPlusOne);) {
VFRange SubRange = {VF, MaxVFPlusOne};
- VPlans.push_back(buildVPlanWithVPRecipes(SubRange, NeedDef,
- DeadInstructions, SinkAfter));
+ VPlans.push_back(
+ buildVPlanWithVPRecipes(SubRange, DeadInstructions, SinkAfter));
VF = SubRange.End;
}
}
VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
- VFRange &Range, SmallPtrSetImpl<Value *> &NeedDef,
- SmallPtrSetImpl<Instruction *> &DeadInstructions,
+ VFRange &Range, SmallPtrSetImpl<Instruction *> &DeadInstructions,
const DenseMap<Instruction *, Instruction *> &SinkAfter) {
// Hold a mapping from predicated instructions to their recipes, in order to
@@ -7767,10 +7749,6 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
VPBasicBlock *VPBB = new VPBasicBlock("Pre-Entry");
Plan->setEntry(VPBB);
- // Represent values that will have defs inside VPlan.
- for (Value *V : NeedDef)
- Plan->addVPValue(V);
-
// Scan the body of the loop in a topological order to visit each basic block
// after having visited its predecessor basic blocks.
LoopBlocksDFS DFS(OrigLoop);
@@ -7800,12 +7778,8 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
// Check if the recipe can be converted to a VPValue. We need the extra
// down-casting step until VPRecipeBase inherits from VPValue.
VPValue *MaybeVPValue = Recipe->toVPValue();
- if (!Instr->getType()->isVoidTy() && MaybeVPValue) {
- if (NeedDef.contains(Instr))
- Plan->addOrReplaceVPValue(Instr, MaybeVPValue);
- else
- Plan->addVPValue(Instr, MaybeVPValue);
- }
+ if (!Instr->getType()->isVoidTy() && MaybeVPValue)
+ Plan->addVPValue(Instr, MaybeVPValue);
RecipeBuilder.setRecipe(Instr, Recipe);
VPBB->appendRecipe(Recipe);
@@ -7974,6 +7948,9 @@ void LoopVectorizationPlanner::adjustRecipesForInLoopReductions(
: nullptr;
VPReductionRecipe *RedRecipe = new VPReductionRecipe(
&RdxDesc, R, ChainOp, VecOp, CondOp, Legal->hasFunNoNaNAttr(), TTI);
+ WidenRecipe->toVPValue()->replaceAllUsesWith(RedRecipe);
+ Plan->removeVPValueFor(R);
+ Plan->addVPValue(R, RedRecipe);
WidenRecipe->getParent()->insert(RedRecipe, WidenRecipe->getIterator());
WidenRecipe->eraseFromParent();
@@ -7983,6 +7960,8 @@ void LoopVectorizationPlanner::adjustRecipesForInLoopReductions(
RecipeBuilder.getRecipe(cast<Instruction>(R->getOperand(0)));
assert(isa<VPWidenRecipe>(CompareRecipe) &&
"Expected to replace a VPWidenSC");
+ assert(CompareRecipe->toVPValue()->getNumUsers() == 0 &&
+ "Expected no remaining users");
CompareRecipe->eraseFromParent();
}
Chain = R;
@@ -8027,7 +8006,7 @@ void VPWidenSelectRecipe::execute(VPTransformState &State) {
}
void VPWidenRecipe::execute(VPTransformState &State) {
- State.ILV->widenInstruction(Ingredient, *this, State);
+ State.ILV->widenInstruction(*getUnderlyingInstr(), this, *this, State);
}
void VPWidenGEPRecipe::execute(VPTransformState &State) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 08ebafbf12cc..e5ffab99af54 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -129,6 +129,8 @@ VPValue *VPRecipeBase::toVPValue() {
return V;
if (auto *V = dyn_cast<VPWidenGEPRecipe>(this))
return V;
+ if (auto *V = dyn_cast<VPWidenRecipe>(this))
+ return V;
return nullptr;
}
@@ -145,6 +147,8 @@ const VPValue *VPRecipeBase::toVPValue() const {
return V;
if (auto *V = dyn_cast<VPWidenGEPRecipe>(this))
return V;
+ if (auto *V = dyn_cast<VPWidenRecipe>(this))
+ return V;
return nullptr;
}
@@ -883,8 +887,10 @@ void VPWidenSelectRecipe::print(raw_ostream &O, const Twine &Indent,
void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
- O << "\"WIDEN\\l\"";
- O << "\" " << VPlanIngredient(&Ingredient);
+ O << "\"WIDEN ";
+ printAsOperand(O, SlotTracker);
+ O << " = " << getUnderlyingInstr()->getOpcodeName() << " ";
+ printOperands(O, SlotTracker);
}
void VPWidenIntOrFpInductionRecipe::print(raw_ostream &O, const Twine &Indent,
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 1de484ab6763..fb7117b4ca8c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -828,14 +828,12 @@ class VPInstruction : public VPUser, public VPValue, public VPRecipeBase {
/// VPWidenRecipe is a recipe for producing a copy of vector type its
/// ingredient. This recipe covers most of the traditional vectorization cases
/// where each ingredient transforms into a vectorized version of itself.
-class VPWidenRecipe : public VPRecipeBase, public VPUser {
- /// Hold the instruction to be widened.
- Instruction &Ingredient;
-
+class VPWidenRecipe : public VPRecipeBase, public VPValue, public VPUser {
public:
template <typename IterT>
VPWidenRecipe(Instruction &I, iterator_range<IterT> Operands)
- : VPRecipeBase(VPWidenSC), VPUser(Operands), Ingredient(I) {}
+ : VPRecipeBase(VPRecipeBase::VPWidenSC), VPValue(VPValue::VPWidenSC, &I),
+ VPUser(Operands) {}
~VPWidenRecipe() override = default;
@@ -843,6 +841,9 @@ class VPWidenRecipe : public VPRecipeBase, public VPUser {
static inline bool classof(const VPRecipeBase *V) {
return V->getVPRecipeID() == VPRecipeBase::VPWidenSC;
}
+ static inline bool classof(const VPValue *V) {
+ return V->getVPValueID() == VPValue::VPWidenSC;
+ }
/// Produce widened copies of all Ingredients.
void execute(VPTransformState &State) override;
@@ -1736,15 +1737,6 @@ class VPlan {
Value2VPValue[V] = VPV;
}
- void addOrReplaceVPValue(Value *V, VPValue *VPV) {
- assert(V && "Trying to add a null Value to VPlan");
- auto I = Value2VPValue.find(V);
- if (I == Value2VPValue.end())
- Value2VPValue[V] = VPV;
- else
- I->second = VPV;
- }
-
VPValue *getVPValue(Value *V) {
assert(V && "Trying to get the VPValue of a null Value");
assert(Value2VPValue.count(V) && "Value does not exist in VPlan");
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 9f4f09f8b19a..0bc40112c9c5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -90,6 +90,7 @@ class VPValue {
VPInstructionSC,
VPMemoryInstructionSC,
VPReductionSC,
+ VPWidenSC,
VPVWidenCallSC,
VPVWidenGEPSC,
VPVWidenSelectSC,
diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
index e8e80d19bc75..dae6a63e9362 100644
--- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
@@ -38,7 +38,7 @@ for.end:
; CHECK: N0 [label =
; CHECK-NEXT: "loop:\n" +
; CHECK-NEXT: "WIDEN-INDUCTION %iv = phi 0, %iv.next\l" +
-; CHECK-NEXT: "WIDEN\l"" %cond0 = icmp %iv, 13\l" +
+; CHECK-NEXT: "WIDEN ir<%cond0> = icmp ir<%iv>, ir<13>\l" +
; CHECK-NEXT: "WIDEN-SELECT ir<%s> = select ir<%cond0>, ir<10>, ir<20>\l"
; CHECK-NEXT: ]
define void @test() {
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
index 957607e0dd00..21d236ac76b2 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
@@ -42,9 +42,9 @@ define void @print_widen_gep_and_select(i64 %n, float* noalias %y, float* noalia
; CHECK-NEXT: "WIDEN-INDUCTION %iv = phi %iv.next, 0\l" +
; CHECK-NEXT: "WIDEN-GEP Inv[Var] ir<%arrayidx> = getelementptr ir<%y>, ir<%iv>\l" +
; CHECK-NEXT: "WIDEN ir<%lv> = load ir<%arrayidx>\l" +
-; CHECK-NEXT: "WIDEN\l"" %cmp = icmp %arrayidx, %z\l" +
+; CHECK-NEXT: "WIDEN ir<%cmp> = icmp ir<%arrayidx>, ir<%z>\l" +
; CHECK-NEXT: "WIDEN-SELECT ir<%sel> = select ir<%cmp>, ir<1.000000e+01>, ir<2.000000e+01>\l" +
-; CHECK-NEXT: "WIDEN\l"" %add = fadd %lv, %sel\l" +
+; CHECK-NEXT: "WIDEN ir<%add> = fadd ir<%lv>, ir<%sel>\l" +
; CHECK-NEXT: "CLONE %arrayidx2 = getelementptr %x, %iv\l" +
; CHECK-NEXT: "WIDEN store ir<%arrayidx2>, ir<%add>\l"
; CHECK-NEXT: ]
More information about the llvm-commits
mailing list