[llvm] 4f07508 - [LLVM][VPlan] Keep all VPBlend masks until VPlan transformation. (#104015)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 21 04:51:43 PDT 2024
Author: Paul Walker
Date: 2024-08-21T12:51:40+01:00
New Revision: 4f075086e7b8d9108749117f53999cd4afdd6894
URL: https://github.com/llvm/llvm-project/commit/4f075086e7b8d9108749117f53999cd4afdd6894
DIFF: https://github.com/llvm/llvm-project/commit/4f075086e7b8d9108749117f53999cd4afdd6894.diff
LOG: [LLVM][VPlan] Keep all VPBlend masks until VPlan transformation. (#104015)
It's not possible to pick the best mask to remove when optimising
VPBlend at construction and so this patch refactors the code to move the
decision (and thus transformation) to VPlanTransforms.
NOTE: This patch does not change the decision of which mask to pick.
That will be done in a following PR to keep this patch as NFC from an
output point of view.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e3049da1b2218..2145bb8c9ca87 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8149,8 +8149,6 @@ VPBlendRecipe *VPRecipeBuilder::tryToBlend(PHINode *Phi,
// builder. At this point we generate the predication tree. There may be
// duplications since this is a simple recursive scan, but future
// optimizations will clean it up.
- // TODO: At the moment the first mask is always skipped, but it would be
- // better to skip the most expensive mask.
SmallVector<VPValue *, 2> OperandsWithMask;
for (unsigned In = 0; In < NumIncoming; In++) {
@@ -8163,8 +8161,6 @@ VPBlendRecipe *VPRecipeBuilder::tryToBlend(PHINode *Phi,
"Distinct incoming values with one having a full mask");
break;
}
- if (In == 0)
- continue;
OperandsWithMask.push_back(EdgeMask);
}
return new VPBlendRecipe(Phi, OperandsWithMask);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index a99f3882092c2..24da8f6700dfa 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2041,12 +2041,12 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe {
class VPBlendRecipe : public VPSingleDefRecipe {
public:
/// The blend operation is a User of the incoming values and of their
- /// respective masks, ordered [I0, I1, M1, I2, M2, ...]. Note that the first
- /// incoming value does not have a mask associated.
+ /// respective masks, ordered [I0, M0, I1, M1, I2, M2, ...]. Note that M0 can
+ /// be ommited (implied by passing an odd number of operands) in which case
+ /// all other incoming values are merged into it.
VPBlendRecipe(PHINode *Phi, ArrayRef<VPValue *> Operands)
: VPSingleDefRecipe(VPDef::VPBlendSC, Operands, Phi, Phi->getDebugLoc()) {
- assert((Operands.size() + 1) % 2 == 0 &&
- "Expected an odd number of operands");
+ assert(Operands.size() > 0 && "Expected at least one operand!");
}
VPBlendRecipe *clone() override {
@@ -2056,19 +2056,25 @@ class VPBlendRecipe : public VPSingleDefRecipe {
VP_CLASSOF_IMPL(VPDef::VPBlendSC)
- /// Return the number of incoming values, taking into account that the first
- /// incoming value has no mask.
- unsigned getNumIncomingValues() const { return (getNumOperands() + 1) / 2; }
+ /// A normalized blend is one that has an odd number of operands, whereby the
+ /// first operand does not have an associated mask.
+ bool isNormalized() const { return getNumOperands() % 2; }
+
+ /// Return the number of incoming values, taking into account when normalized
+ /// the first incoming value will have no mask.
+ unsigned getNumIncomingValues() const {
+ return (getNumOperands() + isNormalized()) / 2;
+ }
/// Return incoming value number \p Idx.
VPValue *getIncomingValue(unsigned Idx) const {
- return Idx == 0 ? getOperand(0) : getOperand(Idx * 2 - 1);
+ return Idx == 0 ? getOperand(0) : getOperand(Idx * 2 - isNormalized());
}
/// Return mask number \p Idx.
VPValue *getMask(unsigned Idx) const {
- assert(Idx > 0 && "First index has no mask associated.");
- return getOperand(Idx * 2);
+ assert((Idx > 0 || !isNormalized()) && "First index has no mask!");
+ return Idx == 0 ? getOperand(1) : getOperand(Idx * 2 + !isNormalized());
}
/// Generate the phi/select nodes.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index aea5e681b081c..63e0e8a898137 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -1703,6 +1703,7 @@ void VPVectorPointerRecipe::print(raw_ostream &O, const Twine &Indent,
#endif
void VPBlendRecipe::execute(VPTransformState &State) {
+ assert(isNormalized() && "Expected blend to be normalized!");
State.setDebugLocFrom(getDebugLoc());
// We know that all PHIs in non-header blocks are converted into
// selects, so we don't have to worry about the insertion order and we
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a2496f067024c..55e90298b36cd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -895,15 +895,47 @@ void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) {
/// Try to simplify recipe \p R.
static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
using namespace llvm::VPlanPatternMatch;
- // Try to remove redundant blend recipes.
+
if (auto *Blend = dyn_cast<VPBlendRecipe>(&R)) {
- VPValue *Inc0 = Blend->getIncomingValue(0);
+ // Try to remove redundant blend recipes.
+ SmallPtrSet<VPValue *, 4> UniqueValues;
+ if (Blend->isNormalized() || !match(Blend->getMask(0), m_False()))
+ UniqueValues.insert(Blend->getIncomingValue(0));
for (unsigned I = 1; I != Blend->getNumIncomingValues(); ++I)
- if (Inc0 != Blend->getIncomingValue(I) &&
- !match(Blend->getMask(I), m_False()))
- return;
- Blend->replaceAllUsesWith(Inc0);
+ if (!match(Blend->getMask(I), m_False()))
+ UniqueValues.insert(Blend->getIncomingValue(I));
+
+ if (UniqueValues.size() == 1) {
+ Blend->replaceAllUsesWith(*UniqueValues.begin());
+ Blend->eraseFromParent();
+ return;
+ }
+
+ if (Blend->isNormalized())
+ return;
+
+ // Normalize the blend so its first incomming value is used as the initial
+ // value with the others blended into it.
+
+ unsigned StartIndex = 0;
+ SmallVector<VPValue *, 4> OperandsWithMask;
+ OperandsWithMask.push_back(Blend->getIncomingValue(StartIndex));
+
+ for (unsigned I = 0; I != Blend->getNumIncomingValues(); ++I) {
+ if (I == StartIndex)
+ continue;
+ OperandsWithMask.push_back(Blend->getIncomingValue(I));
+ OperandsWithMask.push_back(Blend->getMask(I));
+ }
+
+ auto *NewBlend = new VPBlendRecipe(
+ cast<PHINode>(Blend->getUnderlyingValue()), OperandsWithMask);
+ NewBlend->insertBefore(&R);
+
+ VPValue *DeadMask = Blend->getMask(StartIndex);
+ Blend->replaceAllUsesWith(NewBlend);
Blend->eraseFromParent();
+ recursivelyDeleteDeadRecipes(DeadMask);
return;
}
More information about the llvm-commits
mailing list