[llvm] bd0b131 - [VPlan] Turn VPReplicateRecipe into a VPValue.
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Thu Nov 26 05:56:14 PST 2020
Author: Florian Hahn
Date: 2020-11-26T13:50:24Z
New Revision: bd0b1311dbd59b4b7c4188ef0ed930848dc657d1
URL: https://github.com/llvm/llvm-project/commit/bd0b1311dbd59b4b7c4188ef0ed930848dc657d1
DIFF: https://github.com/llvm/llvm-project/commit/bd0b1311dbd59b4b7c4188ef0ed930848dc657d1.diff
LOG: [VPlan] Turn VPReplicateRecipe into a VPValue.
Update VPReplicateRecipe to inherit from VPValue. This still does not
update scalarizeInstruction to set the result for the VPValue of
VPReplicateRecipe, because this first requires tracking scalar values in
VPTransformState.
Reviewed By: reames
Differential Revision: https://reviews.llvm.org/D91500
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanValue.h
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index c9d6a6199476..d72c18061b82 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2674,7 +2674,9 @@ void InnerLoopVectorizer::scalarizeInstruction(Instruction *Instr, VPUser &User,
// Place the cloned scalar in the new loop.
Builder.Insert(Cloned);
- // Add the cloned scalar to the scalar map entry.
+ // TODO: Set result for VPValue of VPReciplicateRecipe. This requires
+ // representing scalar values in VPTransformState. Add the cloned scalar to
+ // the scalar map entry.
VectorLoopValueMap.setScalarValue(Instr, Instance, Cloned);
// If we just cloned a new assumption, add it the assumption cache.
@@ -7549,6 +7551,7 @@ VPBasicBlock *VPRecipeBuilder::handleReplication(
auto *Recipe = new VPReplicateRecipe(I, Plan->mapToVPValues(I->operands()),
IsUniform, IsPredicated);
setRecipe(I, Recipe);
+ Plan->addVPValue(I, Recipe);
// Find if I uses a predicated instruction. If so, it will use its scalar
// value. Avoid hoisting the insert-element which packs the scalar value into
@@ -8103,18 +8106,20 @@ void VPReductionRecipe::execute(VPTransformState &State) {
void VPReplicateRecipe::execute(VPTransformState &State) {
if (State.Instance) { // Generate a single instance.
- State.ILV->scalarizeInstruction(Ingredient, *this, *State.Instance,
- IsPredicated, State);
+ State.ILV->scalarizeInstruction(getUnderlyingInstr(), *this,
+ *State.Instance, IsPredicated, State);
// Insert scalar instance packing it into a vector.
if (AlsoPack && State.VF.isVector()) {
// If we're constructing lane 0, initialize to start from undef.
if (State.Instance->Lane == 0) {
assert(!State.VF.isScalable() && "VF is assumed to be non scalable.");
- Value *Undef =
- UndefValue::get(VectorType::get(Ingredient->getType(), State.VF));
- State.ValueMap.setVectorValue(Ingredient, State.Instance->Part, Undef);
+ Value *Undef = UndefValue::get(
+ VectorType::get(getUnderlyingValue()->getType(), State.VF));
+ State.ValueMap.setVectorValue(getUnderlyingInstr(),
+ State.Instance->Part, Undef);
}
- State.ILV->packScalarIntoVectorValue(Ingredient, *State.Instance);
+ State.ILV->packScalarIntoVectorValue(getUnderlyingInstr(),
+ *State.Instance);
}
return;
}
@@ -8125,7 +8130,7 @@ void VPReplicateRecipe::execute(VPTransformState &State) {
unsigned EndLane = IsUniform ? 1 : State.VF.getKnownMinValue();
for (unsigned Part = 0; Part < State.UF; ++Part)
for (unsigned Lane = 0; Lane < EndLane; ++Lane)
- State.ILV->scalarizeInstruction(Ingredient, *this, {Part, Lane},
+ State.ILV->scalarizeInstruction(getUnderlyingInstr(), *this, {Part, Lane},
IsPredicated, State);
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index e5ffab99af54..3b26d6b52efd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -131,6 +131,8 @@ VPValue *VPRecipeBase::toVPValue() {
return V;
if (auto *V = dyn_cast<VPWidenRecipe>(this))
return V;
+ if (auto *V = dyn_cast<VPReplicateRecipe>(this))
+ return V;
return nullptr;
}
@@ -149,6 +151,8 @@ const VPValue *VPRecipeBase::toVPValue() const {
return V;
if (auto *V = dyn_cast<VPWidenRecipe>(this))
return V;
+ if (auto *V = dyn_cast<VPReplicateRecipe>(this))
+ return V;
return nullptr;
}
@@ -233,14 +237,8 @@ VPBlockBase *VPBlockBase::getEnclosingBlockWithPredecessors() {
void VPBlockBase::deleteCFG(VPBlockBase *Entry) {
SmallVector<VPBlockBase *, 8> Blocks;
- VPValue DummyValue;
- for (VPBlockBase *Block : depth_first(Entry)) {
- // Drop all references in VPBasicBlocks and replace all uses with
- // DummyValue.
- if (auto *VPBB = dyn_cast<VPBasicBlock>(Block))
- VPBB->dropAllReferences(&DummyValue);
+ for (VPBlockBase *Block : depth_first(Entry))
Blocks.push_back(Block);
- }
for (VPBlockBase *Block : Blocks)
delete Block;
@@ -384,6 +382,13 @@ void VPBasicBlock::dropAllReferences(VPValue *NewValue) {
}
}
+void VPRegionBlock::dropAllReferences(VPValue *NewValue) {
+ for (VPBlockBase *Block : depth_first(Entry))
+ // Drop all references in VPBasicBlocks and replace all uses with
+ // DummyValue.
+ Block->dropAllReferences(NewValue);
+}
+
void VPRegionBlock::execute(VPTransformState *State) {
ReversePostOrderTraversal<VPBlockBase *> RPOT(Entry);
@@ -962,7 +967,7 @@ void VPReductionRecipe::print(raw_ostream &O, const Twine &Indent,
void VPReplicateRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << "\"" << (IsUniform ? "CLONE " : "REPLICATE ")
- << VPlanIngredient(Ingredient);
+ << VPlanIngredient(getUnderlyingInstr());
if (AlsoPack)
O << " (S->V)";
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 28595b9ec408..b4d84ed57773 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -614,6 +614,10 @@ class VPBlockBase {
// hoisted into a VPBlockBase.
return true;
}
+
+ /// Replace all operands of VPUsers in the block with \p NewValue and also
+ /// replaces all uses of VPValues defined in the block with NewValue.
+ virtual void dropAllReferences(VPValue *NewValue) = 0;
};
/// VPRecipeBase is a base class modeling a sequence of one or more output IR
@@ -1124,10 +1128,7 @@ class VPReductionRecipe : public VPRecipeBase, public VPValue, public VPUser {
/// copies of the original scalar type, one per lane, instead of producing a
/// single copy of widened type for all lanes. If the instruction is known to be
/// uniform only one copy, per lane zero, will be generated.
-class VPReplicateRecipe : public VPRecipeBase, public VPUser {
- /// The instruction being replicated.
- Instruction *Ingredient;
-
+class VPReplicateRecipe : public VPRecipeBase, public VPUser, public VPValue {
/// Indicator if only a single replica per lane is needed.
bool IsUniform;
@@ -1141,8 +1142,9 @@ class VPReplicateRecipe : public VPRecipeBase, public VPUser {
template <typename IterT>
VPReplicateRecipe(Instruction *I, iterator_range<IterT> Operands,
bool IsUniform, bool IsPredicated = false)
- : VPRecipeBase(VPReplicateSC), VPUser(Operands), Ingredient(I),
- IsUniform(IsUniform), IsPredicated(IsPredicated) {
+ : VPRecipeBase(VPReplicateSC), VPUser(Operands),
+ VPValue(VPVReplicateSC, I), IsUniform(IsUniform),
+ IsPredicated(IsPredicated) {
// Retain the previous behavior of predicateInstructions(), where an
// insert-element of a predicated instruction got hoisted into the
// predicated basic block iff it was its only user. This is achieved by
@@ -1158,6 +1160,10 @@ class VPReplicateRecipe : public VPRecipeBase, public VPUser {
return V->getVPRecipeID() == VPRecipeBase::VPReplicateSC;
}
+ static inline bool classof(const VPValue *V) {
+ return V->getVPValueID() == VPValue::VPVReplicateSC;
+ }
+
/// Generate replicas of the desired Ingredient. Replicas will be generated
/// for all parts and lanes unless a specific part and lane are specified in
/// the \p State.
@@ -1168,6 +1174,8 @@ class VPReplicateRecipe : public VPRecipeBase, public VPUser {
/// Print the recipe.
void print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const override;
+
+ bool isUniform() const { return IsUniform; }
};
/// A recipe for generating conditional branches on the bits of a mask.
@@ -1407,13 +1415,11 @@ class VPBasicBlock : public VPBlockBase {
/// this VPBasicBlock, thereby "executing" the VPlan.
void execute(struct VPTransformState *State) override;
- /// Replace all operands of VPUsers in the block with \p NewValue and also
- /// replaces all uses of VPValues defined in the block with NewValue.
- void dropAllReferences(VPValue *NewValue);
-
/// Return the position of the first non-phi node recipe in the block.
iterator getFirstNonPhi();
+ void dropAllReferences(VPValue *NewValue) override;
+
private:
/// Create an IR BasicBlock to hold the output instructions generated by this
/// VPBasicBlock, and return it. Update the CFGState accordingly.
@@ -1454,8 +1460,11 @@ class VPRegionBlock : public VPBlockBase {
IsReplicator(IsReplicator) {}
~VPRegionBlock() override {
- if (Entry)
+ if (Entry) {
+ VPValue DummyValue;
+ Entry->dropAllReferences(&DummyValue);
deleteCFG(Entry);
+ }
}
/// Method to support type inquiry through isa, cast, and dyn_cast.
@@ -1500,6 +1509,8 @@ class VPRegionBlock : public VPBlockBase {
/// The method which generates the output IR instructions that correspond to
/// this VPRegionBlock, thereby "executing" the VPlan.
void execute(struct VPTransformState *State) override;
+
+ void dropAllReferences(VPValue *NewValue) override;
};
//===----------------------------------------------------------------------===//
@@ -1674,8 +1685,13 @@ class VPlan {
}
~VPlan() {
- if (Entry)
+ if (Entry) {
+ VPValue DummyValue;
+ for (VPBlockBase *Block : depth_first(Entry))
+ Block->dropAllReferences(&DummyValue);
+
VPBlockBase::deleteCFG(Entry);
+ }
for (VPValue *VPV : VPValuesToFree)
delete VPV;
if (BackedgeTakenCount)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 9561abd10b5b..5cb818e13d07 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -90,6 +90,7 @@ class VPValue {
VPVInstructionSC,
VPVMemoryInstructionSC,
VPVReductionSC,
+ VPVReplicateSC,
VPVWidenSC,
VPVWidenCallSC,
VPVWidenGEPSC,
More information about the llvm-commits
mailing list