[llvm] e2a1867 - [LV] Add VPValue operands to VPBlendRecipe (NFCI)
Gil Rapaport via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 9 08:58:21 PDT 2020
Author: Gil Rapaport
Date: 2020-04-09T18:48:33+03:00
New Revision: e2a186788051ac3fb22f8d99bec4b68ee3d7b81b
URL: https://github.com/llvm/llvm-project/commit/e2a186788051ac3fb22f8d99bec4b68ee3d7b81b
DIFF: https://github.com/llvm/llvm-project/commit/e2a186788051ac3fb22f8d99bec4b68ee3d7b81b.diff
LOG: [LV] Add VPValue operands to VPBlendRecipe (NFCI)
InnerLoopVectorizer's code called during VPlan execution still relies on
original IR's def-use relations to decide which vector code to generate,
limiting VPlan transformations ability to modify def-use relations and still
have ILV generate the vector code.
This commit introduces VPValues for VPBlendRecipe to use as the values to
blend. The recipe is generated with VPValues wrapping the phi's incoming values
of the scalar phi. This reduces ingredient def-use usage by ILV as a step
towards full VPlan-based def-use relations.
Differential Revision: https://reviews.llvm.org/D77539
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 49968ee0adcf..4b7e177a9edd 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6884,17 +6884,18 @@ VPBlendRecipe *VPRecipeBuilder::tryToBlend(Instruction *I, VPlanPtr &Plan) {
// duplications since this is a simple recursive scan, but future
// optimizations will clean it up.
- SmallVector<VPValue *, 2> Masks;
+ SmallVector<VPValue *, 2> Operands;
unsigned NumIncoming = Phi->getNumIncomingValues();
for (unsigned In = 0; In < NumIncoming; In++) {
VPValue *EdgeMask =
createEdgeMask(Phi->getIncomingBlock(In), Phi->getParent(), Plan);
assert((EdgeMask || NumIncoming == 1) &&
"Multiple predecessors with one having a full mask");
+ Operands.push_back(Plan->getOrAddVPValue(Phi->getIncomingValue(In)));
if (EdgeMask)
- Masks.push_back(EdgeMask);
+ Operands.push_back(EdgeMask);
}
- return new VPBlendRecipe(Phi, Masks);
+ return new VPBlendRecipe(Phi, Operands);
}
VPWidenCallRecipe *
@@ -7436,10 +7437,8 @@ void VPBlendRecipe::execute(VPTransformState &State) {
// duplications since this is a simple recursive scan, but future
// optimizations will clean it up.
- unsigned NumIncoming = Phi->getNumIncomingValues();
+ unsigned NumIncoming = getNumIncomingValues();
- assert((User || NumIncoming == 1) &&
- "Multiple predecessors with predecessors having a full mask");
// Generate a sequence of selects of the form:
// SELECT(Mask3, In3,
// SELECT(Mask2, In2,
@@ -7449,14 +7448,13 @@ void VPBlendRecipe::execute(VPTransformState &State) {
for (unsigned Part = 0; Part < State.UF; ++Part) {
// We might have single edge PHIs (blocks) - use an identity
// 'select' for the first PHI operand.
- Value *In0 =
- State.ILV->getOrCreateVectorValue(Phi->getIncomingValue(In), Part);
+ Value *In0 = State.get(getIncomingValue(In), Part);
if (In == 0)
Entry[Part] = In0; // Initialize with the first incoming value.
else {
// Select between the current value and the previous incoming edge
// based on the incoming mask.
- Value *Cond = State.get(User->getOperand(In), Part);
+ Value *Cond = State.get(getMask(In), Part);
Entry[Part] =
State.Builder.CreateSelect(Cond, In0, Entry[Part], "predphi");
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index dab4cbbd2423..ae295bdcac66 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -756,17 +756,17 @@ void VPBlendRecipe::print(raw_ostream &O, const Twine &Indent,
O << " +\n" << Indent << "\"BLEND ";
Phi->printAsOperand(O, false);
O << " =";
- if (!User) {
+ if (getNumIncomingValues() == 1) {
// Not a User of any mask: not really blending, this is a
// single-predecessor phi.
O << " ";
- Phi->getIncomingValue(0)->printAsOperand(O, false);
+ getIncomingValue(0)->printAsOperand(O, SlotTracker);
} else {
- for (unsigned I = 0, E = User->getNumOperands(); I < E; ++I) {
+ for (unsigned I = 0, E = getNumIncomingValues(); I < E; ++I) {
O << " ";
- Phi->getIncomingValue(I)->printAsOperand(O, false);
+ getIncomingValue(I)->printAsOperand(O, SlotTracker);
O << "/";
- User->getOperand(I)->printAsOperand(O, SlotTracker);
+ getMask(I)->printAsOperand(O, SlotTracker);
}
}
O << "\\l\"";
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 55c0754b9f95..d331992e534c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -902,17 +902,18 @@ class VPBlendRecipe : public VPRecipeBase {
private:
PHINode *Phi;
- /// The blend operation is a User of a mask, if not null.
- std::unique_ptr<VPUser> User;
+ /// The blend operation is a User of the incoming values and of their
+ /// respective masks, ordered [I0, M0, I1, M1, ...]. Note that a single value
+ /// would be incoming with a full mask for which there is no VPValue.
+ VPUser User;
public:
- VPBlendRecipe(PHINode *Phi, ArrayRef<VPValue *> Masks)
- : VPRecipeBase(VPBlendSC), Phi(Phi) {
- assert((Phi->getNumIncomingValues() == 1 ||
- Phi->getNumIncomingValues() == Masks.size()) &&
- "Expected the same number of incoming values and masks");
- if (!Masks.empty())
- User.reset(new VPUser(Masks));
+ VPBlendRecipe(PHINode *Phi, ArrayRef<VPValue *> Operands)
+ : VPRecipeBase(VPBlendSC), Phi(Phi), User(Operands) {
+ assert(((Operands.size() == 1) ||
+ (Operands.size() > 2 && Operands.size() % 2 == 0)) &&
+ "Expected either a single incoming value or a greater than two and "
+ "even number of operands");
}
/// Method to support type inquiry through isa, cast, and dyn_cast.
@@ -920,6 +921,20 @@ class VPBlendRecipe : public VPRecipeBase {
return V->getVPRecipeID() == VPRecipeBase::VPBlendSC;
}
+ /// Return the number of incoming values, taking into account that a single
+ /// incoming value has no mask.
+ unsigned getNumIncomingValues() const {
+ return (User.getNumOperands() + 1) / 2;
+ }
+
+ /// Return incoming value number \p Idx.
+ VPValue *getIncomingValue(unsigned Idx) const {
+ return User.getOperand(Idx * 2);
+ }
+
+ /// Return mask number \p Idx.
+ VPValue *getMask(unsigned Idx) const { return User.getOperand(Idx * 2 + 1); }
+
/// Generate the phi/select nodes.
void execute(VPTransformState &State) override;
More information about the llvm-commits
mailing list