[llvm] 6c3451c - [VPlan] Add VPReductionPHIRecipe (NFC).
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 6 03:25:48 PDT 2021
Author: Florian Hahn
Date: 2021-07-06T11:25:28+01:00
New Revision: 6c3451cd76cbd0cd973d9c2b08b168dcd0bce3c2
URL: https://github.com/llvm/llvm-project/commit/6c3451cd76cbd0cd973d9c2b08b168dcd0bce3c2
DIFF: https://github.com/llvm/llvm-project/commit/6c3451cd76cbd0cd973d9c2b08b168dcd0bce3c2.diff
LOG: [VPlan] Add VPReductionPHIRecipe (NFC).
This patch is a first step towards splitting up VPWidenPHIRecipe into
separate recipes for the 3 distinct cases they model:
1. reduction phis,
2. first-order recurrence phis,
3. pointer induction phis.
This allows untangling the code generation and allows us to reduce the
reliance on LoopVectorizationCostModel during VPlan code generation.
Discussed/suggested in D100102, D100113, D104197.
Reviewed By: Ayal
Differential Revision: https://reviews.llvm.org/D104989
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanValue.h
llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
llvm/test/Transforms/LoopVectorize/vplan-printing.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index f3e5b020e8b6..13b8216862e6 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -503,11 +503,11 @@ class InnerLoopVectorizer {
unsigned UF, ElementCount VF, bool IsPtrLoopInvariant,
SmallBitVector &IsIndexLoopInvariant, VPTransformState &State);
- /// Vectorize a single PHINode in a block. This method handles the induction
- /// variable canonicalization. It supports both VF = 1 for unrolled loops and
- /// arbitrary length vectors.
- void widenPHIInstruction(Instruction *PN, RecurrenceDescriptor *RdxDesc,
- VPWidenPHIRecipe *PhiR, VPTransformState &State);
+ /// Vectorize a single first-order recurrence or pointer induction PHINode in
+ /// a block. This method handles the induction variable canonicalization. It
+ /// supports both VF = 1 for unrolled loops and arbitrary length vectors.
+ void widenPHIInstruction(Instruction *PN, VPWidenPHIRecipe *PhiR,
+ VPTransformState &State);
/// A helper function to scalarize a single Instruction in the innermost loop.
/// Generates a sequence of scalar instances for each lane between \p MinLane
@@ -596,7 +596,7 @@ class InnerLoopVectorizer {
/// Fix a reduction cross-iteration phi. This is the second phase of
/// vectorizing this phi node.
- void fixReduction(VPWidenPHIRecipe *Phi, VPTransformState &State);
+ void fixReduction(VPReductionPHIRecipe *Phi, VPTransformState &State);
/// Clear NSW/NUW flags from reduction instructions if necessary.
void clearReductionWrapFlags(const RecurrenceDescriptor &RdxDesc,
@@ -4135,8 +4135,8 @@ void InnerLoopVectorizer::fixCrossIterationPHIs(VPTransformState &State) {
if (!PhiR)
continue;
auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
- if (PhiR->getRecurrenceDescriptor()) {
- fixReduction(PhiR, State);
+ if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(PhiR)) {
+ fixReduction(ReductionPhi, State);
} else if (Legal->isFirstOrderRecurrence(OrigPhi))
fixFirstOrderRecurrence(PhiR, State);
}
@@ -4320,19 +4320,18 @@ void InnerLoopVectorizer::fixFirstOrderRecurrence(VPWidenPHIRecipe *PhiR,
LCSSAPhi.addIncoming(ExtractForPhiUsedOutsideLoop, LoopMiddleBlock);
}
-void InnerLoopVectorizer::fixReduction(VPWidenPHIRecipe *PhiR,
+void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
VPTransformState &State) {
PHINode *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
// Get it's reduction variable descriptor.
assert(Legal->isReductionVariable(OrigPhi) &&
"Unable to find the reduction variable");
- const RecurrenceDescriptor &RdxDesc = *PhiR->getRecurrenceDescriptor();
+ const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
RecurKind RK = RdxDesc.getRecurrenceKind();
TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue();
Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();
setDebugLocFromInst(ReductionStartValue);
- bool IsInLoopReductionPhi = Cost->isInLoopReduction(OrigPhi);
VPValue *LoopExitInstDef = State.Plan->getVPValue(LoopExitInst);
// This is the vector-clone of the value that leaves the loop.
@@ -4347,14 +4346,11 @@ void InnerLoopVectorizer::fixReduction(VPWidenPHIRecipe *PhiR,
// any loop invariant values.
BasicBlock *VectorLoopLatch = LI->getLoopFor(LoopVectorBody)->getLoopLatch();
- bool IsOrdered = IsInLoopReductionPhi && Cost->useOrderedReductions(RdxDesc);
-
- for (unsigned Part = 0; Part < UF; ++Part) {
- if (IsOrdered && Part > 0)
- break;
+ unsigned LastPartForNewPhi = PhiR->isOrdered() ? 1 : UF;
+ for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
Value *VecRdxPhi = State.get(PhiR->getVPSingleValue(), Part);
Value *Val = State.get(PhiR->getBackedgeValue(), Part);
- if (IsOrdered)
+ if (PhiR->isOrdered())
Val = State.get(PhiR->getBackedgeValue(), UF - 1);
cast<PHINode>(VecRdxPhi)->addIncoming(Val, VectorLoopLatch);
@@ -4373,7 +4369,7 @@ void InnerLoopVectorizer::fixReduction(VPWidenPHIRecipe *PhiR,
// a Select choosing between the vectorized LoopExitInst and vectorized Phi,
// instead of the former. For an inloop reduction the reduction will already
// be predicated, and does not need to be handled here.
- if (Cost->foldTailByMasking() && !IsInLoopReductionPhi) {
+ if (Cost->foldTailByMasking() && !PhiR->isInLoop()) {
for (unsigned Part = 0; Part < UF; ++Part) {
Value *VecLoopExitInst = State.get(LoopExitInstDef, Part);
Value *Sel = nullptr;
@@ -4408,7 +4404,7 @@ void InnerLoopVectorizer::fixReduction(VPWidenPHIRecipe *PhiR,
// then extend the loop exit value to enable InstCombine to evaluate the
// entire expression in the smaller type.
if (VF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) {
- assert(!IsInLoopReductionPhi && "Unexpected truncated inloop reduction!");
+ assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), VF);
Builder.SetInsertPoint(
LI->getLoopFor(LoopVectorBody)->getLoopLatch()->getTerminator());
@@ -4446,7 +4442,7 @@ void InnerLoopVectorizer::fixReduction(VPWidenPHIRecipe *PhiR,
// terminate on this line. This is the easiest way to ensure we don't
// accidentally cause an extra step back into the loop while debugging.
setDebugLocFromInst(LoopMiddleBlock->getTerminator());
- if (IsOrdered)
+ if (PhiR->isOrdered())
ReducedPartRdx = State.get(LoopExitInstDef, UF - 1);
else {
// Floating-point operations should have some FMF to enable the reduction.
@@ -4465,7 +4461,7 @@ void InnerLoopVectorizer::fixReduction(VPWidenPHIRecipe *PhiR,
// Create the reduction after the loop. Note that inloop reductions create the
// target reduction in the loop using a Reduction recipe.
- if (VF.isVector() && !IsInLoopReductionPhi) {
+ if (VF.isVector() && !PhiR->isInLoop()) {
ReducedPartRdx =
createTargetReduction(Builder, TTI, RdxDesc, ReducedPartRdx);
// If the reduction can be performed in a smaller type, we need to extend
@@ -4729,7 +4725,6 @@ void InnerLoopVectorizer::widenGEP(GetElementPtrInst *GEP, VPValue *VPDef,
}
void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN,
- RecurrenceDescriptor *RdxDesc,
VPWidenPHIRecipe *PhiR,
VPTransformState &State) {
PHINode *P = cast<PHINode>(PN);
@@ -4755,68 +4750,21 @@ void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN,
// Phi nodes have cycles, so we need to vectorize them in two stages. This is
// stage #1: We create a new vector PHI node with no incoming edges. We'll use
// this value when we vectorize all of the instructions that use the PHI.
- if (RdxDesc || Legal->isFirstOrderRecurrence(P)) {
- bool ScalarPHI =
- (State.VF.isScalar()) || Cost->isInLoopReduction(cast<PHINode>(PN));
- Type *VecTy =
- ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), State.VF);
-
- bool IsOrdered = Cost->isInLoopReduction(cast<PHINode>(PN)) &&
- Cost->useOrderedReductions(*RdxDesc);
- unsigned LastPartForNewPhi = IsOrdered ? 1 : State.UF;
- for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
+ if (Legal->isFirstOrderRecurrence(P)) {
+ Type *VecTy = State.VF.isScalar()
+ ? PN->getType()
+ : VectorType::get(PN->getType(), State.VF);
+
+ for (unsigned Part = 0; Part < State.UF; ++Part) {
Value *EntryPart = PHINode::Create(
VecTy, 2, "vec.phi", &*LoopVectorBody->getFirstInsertionPt());
State.set(PhiR, EntryPart, Part);
}
- if (Legal->isFirstOrderRecurrence(P))
return;
- VPValue *StartVPV = PhiR->getStartValue();
- Value *StartV = StartVPV->getLiveInIRValue();
-
- Value *Iden = nullptr;
-
- assert(Legal->isReductionVariable(P) && StartV &&
- "RdxDesc should only be set for reduction variables; in that case "
- "a StartV is also required");
- RecurKind RK = RdxDesc->getRecurrenceKind();
- if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) {
- // MinMax reduction have the start value as their identify.
- if (ScalarPHI) {
- Iden = StartV;
- } else {
- IRBuilderBase::InsertPointGuard IPBuilder(Builder);
- Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
- StartV = Iden =
- Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident");
- }
- } else {
- Constant *IdenC = RecurrenceDescriptor::getRecurrenceIdentity(
- RK, VecTy->getScalarType(), RdxDesc->getFastMathFlags());
- Iden = IdenC;
-
- if (!ScalarPHI) {
- Iden = ConstantVector::getSplat(State.VF, IdenC);
- IRBuilderBase::InsertPointGuard IPBuilder(Builder);
- Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
- Constant *Zero = Builder.getInt32(0);
- StartV = Builder.CreateInsertElement(Iden, StartV, Zero);
- }
- }
-
- for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
- Value *EntryPart = State.get(PhiR, Part);
- // Make sure to add the reduction start value only to the
- // first unroll part.
- Value *StartVal = (Part == 0) ? StartV : Iden;
- cast<PHINode>(EntryPart)->addIncoming(StartVal, LoopVectorPreHeader);
- }
-
- return;
}
assert(!Legal->isReductionVariable(P) &&
- "reductions should be handled above");
+ "reductions should be handled elsewhere");
setDebugLocFromInst(P);
@@ -8978,7 +8926,9 @@ VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
RecurrenceDescriptor &RdxDesc = Legal->getReductionVars()[Phi];
assert(RdxDesc.getRecurrenceStartValue() ==
Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()));
- PhiRecipe = new VPWidenPHIRecipe(Phi, RdxDesc, *StartV);
+ PhiRecipe = new VPReductionPHIRecipe(Phi, RdxDesc, *StartV,
+ CM.isInLoopReduction(Phi),
+ CM.useOrderedReductions(RdxDesc));
} else {
PhiRecipe = new VPWidenPHIRecipe(Phi, *StartV);
}
@@ -9493,8 +9443,8 @@ void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
}
void VPWidenPHIRecipe::execute(VPTransformState &State) {
- State.ILV->widenPHIInstruction(cast<PHINode>(getUnderlyingValue()), RdxDesc,
- this, State);
+ State.ILV->widenPHIInstruction(cast<PHINode>(getUnderlyingValue()), this,
+ State);
}
void VPBlendRecipe::execute(VPTransformState &State) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 636b70272e44..f8a41abddb5d 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -763,6 +763,7 @@ void VPlan::execute(VPTransformState *State) {
State->VPValue2Value[Entry.second] = Entry.first;
BasicBlock *VectorPreHeaderBB = State->CFG.PrevBB;
+ State->CFG.VectorPreHeader = VectorPreHeaderBB;
BasicBlock *VectorHeaderBB = VectorPreHeaderBB->getSingleSuccessor();
assert(VectorHeaderBB && "Loop preheader does not have a single successor.");
@@ -1114,6 +1115,74 @@ void VPWidenPHIRecipe::print(raw_ostream &O, const Twine &Indent,
printOperands(O, SlotTracker);
}
+void VPReductionPHIRecipe::print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const {
+ O << Indent << "WIDEN-REDUCTION-PHI ";
+
+ printAsOperand(O, SlotTracker);
+ O << " = phi ";
+ printOperands(O, SlotTracker);
+}
+
+void VPReductionPHIRecipe::execute(VPTransformState &State) {
+ PHINode *PN = cast<PHINode>(getUnderlyingValue());
+ auto &Builder = State.Builder;
+
+ // In order to support recurrences we need to be able to vectorize Phi nodes.
+ // Phi nodes have cycles, so we need to vectorize them in two stages. This is
+ // stage #1: We create a new vector PHI node with no incoming edges. We'll use
+ // this value when we vectorize all of the instructions that use the PHI.
+ bool ScalarPHI = State.VF.isScalar() || IsInLoop;
+ Type *VecTy =
+ ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), State.VF);
+
+ BasicBlock *HeaderBB = State.CFG.PrevBB;
+ assert(State.LI->getLoopFor(HeaderBB)->getHeader() == HeaderBB &&
+ "recipe must be in the vector loop header");
+ unsigned LastPartForNewPhi = isOrdered() ? 1 : State.UF;
+ for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
+ Value *EntryPart =
+ PHINode::Create(VecTy, 2, "vec.phi", &*HeaderBB->getFirstInsertionPt());
+ State.set(this, EntryPart, Part);
+ }
+ VPValue *StartVPV = getStartValue();
+ Value *StartV = StartVPV->getLiveInIRValue();
+
+ Value *Iden = nullptr;
+ RecurKind RK = RdxDesc.getRecurrenceKind();
+ if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) {
+ // MinMax reduction have the start value as their identify.
+ if (ScalarPHI) {
+ Iden = StartV;
+ } else {
+ IRBuilderBase::InsertPointGuard IPBuilder(Builder);
+ Builder.SetInsertPoint(State.CFG.VectorPreHeader->getTerminator());
+ StartV = Iden =
+ Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident");
+ }
+ } else {
+ Constant *IdenC = RecurrenceDescriptor::getRecurrenceIdentity(
+ RK, VecTy->getScalarType(), RdxDesc.getFastMathFlags());
+ Iden = IdenC;
+
+ if (!ScalarPHI) {
+ Iden = ConstantVector::getSplat(State.VF, IdenC);
+ IRBuilderBase::InsertPointGuard IPBuilder(Builder);
+ Builder.SetInsertPoint(State.CFG.VectorPreHeader->getTerminator());
+ Constant *Zero = Builder.getInt32(0);
+ StartV = Builder.CreateInsertElement(Iden, StartV, Zero);
+ }
+ }
+
+ for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
+ Value *EntryPart = State.get(this, Part);
+ // Make sure to add the reduction start value only to the
+ // first unroll part.
+ Value *StartVal = (Part == 0) ? StartV : Iden;
+ cast<PHINode>(EntryPart)->addIncoming(StartVal, State.CFG.VectorPreHeader);
+ }
+}
+
void VPBlendRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << Indent << "BLEND ";
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index b00c6156f9f6..3f5f066ac27b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -306,6 +306,13 @@ struct VPTransformState {
/// BasicBlock, used for placing the newly created BasicBlocks.
BasicBlock *LastBB = nullptr;
+ /// The IR BasicBlock that is the preheader of the vector loop in the output
+ /// IR.
+ /// FIXME: The vector preheader should also be modeled in VPlan, so any code
+ /// that needs to be added to the preheader gets directly generated by
+ /// VPlan. There should be no need to manage a pointer to the IR BasicBlock.
+ BasicBlock *VectorPreHeader = nullptr;
+
/// A mapping of each VPBasicBlock to the corresponding BasicBlock. In case
/// of replication, maps the BasicBlock of the last replica created.
SmallDenseMap<VPBasicBlock *, BasicBlock *> VPBB2IRBB;
@@ -730,8 +737,7 @@ class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
/// Returns true for PHI-like recipes.
bool isPhi() const {
- return getVPDefID() == VPWidenIntOrFpInductionSC || getVPDefID() == VPWidenPHISC ||
- getVPDefID() == VPPredInstPHISC || getVPDefID() == VPWidenCanonicalIVSC;
+ return getVPDefID() >= VPFirstPHISC && getVPDefID() <= VPLastPHISC;
}
/// Returns true if the recipe may read from memory.
@@ -1044,45 +1050,39 @@ class VPWidenIntOrFpInductionRecipe : public VPRecipeBase {
}
};
-/// A recipe for handling all phi nodes except for integer and FP inductions.
-/// For reduction PHIs, RdxDesc must point to the corresponding recurrence
-/// descriptor. For reductions and first-order recurrences, the start value is
-/// the first operand of the recipe and the incoming value from the backedge is
-/// the second operand. In the VPlan native path, all incoming VPValues &
-/// VPBasicBlock pairs are managed in the recipe directly.
+/// A recipe for handling first order recurrences and pointer inductions. For
+/// first-order recurrences, the start value is the first operand of the recipe
+/// and the incoming value from the backedge is the second operand. It also
+/// serves as base class for VPReductionPHIRecipe. In the VPlan native path, all
+/// incoming VPValues & VPBasicBlock pairs are managed in the recipe directly.
class VPWidenPHIRecipe : public VPRecipeBase, public VPValue {
- /// Descriptor for a reduction PHI.
- RecurrenceDescriptor *RdxDesc = nullptr;
-
/// List of incoming blocks. Only used in the VPlan native path.
SmallVector<VPBasicBlock *, 2> IncomingBlocks;
+protected:
+ VPWidenPHIRecipe(unsigned char VPVID, unsigned char VPDefID, PHINode *Phi)
+ : VPRecipeBase(VPDefID, {}), VPValue(VPVID, Phi, this) {}
+
public:
- /// Create a new VPWidenPHIRecipe for the reduction \p Phi described by \p
- /// RdxDesc.
- VPWidenPHIRecipe(PHINode *Phi, RecurrenceDescriptor &RdxDesc, VPValue &Start)
- : VPWidenPHIRecipe(Phi) {
- this->RdxDesc = &RdxDesc;
- addOperand(&Start);
- }
+ /// Create a VPWidenPHIRecipe for \p Phi
+ VPWidenPHIRecipe(PHINode *Phi)
+ : VPWidenPHIRecipe(VPVWidenPHISC, VPWidenPHISC, Phi) {}
/// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start.
VPWidenPHIRecipe(PHINode *Phi, VPValue &Start) : VPWidenPHIRecipe(Phi) {
addOperand(&Start);
}
- /// Create a VPWidenPHIRecipe for \p Phi
- VPWidenPHIRecipe(PHINode *Phi)
- : VPRecipeBase(VPWidenPHISC, {}),
- VPValue(VPValue::VPVWidenPHISC, Phi, this) {}
~VPWidenPHIRecipe() override = default;
/// Method to support type inquiry through isa, cast, and dyn_cast.
- static inline bool classof(const VPDef *D) {
- return D->getVPDefID() == VPRecipeBase::VPWidenPHISC;
+ static inline bool classof(const VPRecipeBase *B) {
+ return B->getVPDefID() == VPRecipeBase::VPWidenPHISC ||
+ B->getVPDefID() == VPRecipeBase::VPReductionPHISC;
}
static inline bool classof(const VPValue *V) {
- return V->getVPValueID() == VPValue::VPVWidenPHISC;
+ return V->getVPValueID() == VPValue::VPVWidenPHISC ||
+ V->getVPValueID() == VPValue::VPVReductionPHISC;
}
/// Generate the phi/select nodes.
@@ -1117,8 +1117,60 @@ class VPWidenPHIRecipe : public VPRecipeBase, public VPValue {
/// Returns the \p I th incoming VPBasicBlock.
VPBasicBlock *getIncomingBlock(unsigned I) { return IncomingBlocks[I]; }
+};
+
+/// A recipe for handling reduction phis. The start value is the first operand
+/// of the recipe and the incoming value from the backedge is the second
+/// operand.
+class VPReductionPHIRecipe : public VPWidenPHIRecipe {
+ /// Descriptor for the reduction.
+ RecurrenceDescriptor &RdxDesc;
+
+ /// The phi is part of an in-loop reduction.
+ bool IsInLoop;
+
+ /// The phi is part of an ordered reduction. Requires IsInLoop to be true.
+ bool IsOrdered;
+
+public:
+ /// Create a new VPReductionPHIRecipe for the reduction \p Phi described by \p
+ /// RdxDesc.
+ VPReductionPHIRecipe(PHINode *Phi, RecurrenceDescriptor &RdxDesc,
+ VPValue &Start, bool IsInLoop = false,
+ bool IsOrdered = false)
+ : VPWidenPHIRecipe(VPVReductionPHISC, VPReductionPHISC, Phi),
+ RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered) {
+ assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop");
+ addOperand(&Start);
+ }
- RecurrenceDescriptor *getRecurrenceDescriptor() { return RdxDesc; }
+ /// Method to support type inquiry through isa, cast, and dyn_cast.
+ static inline bool classof(const VPRecipeBase *R) {
+ return R->getVPDefID() == VPRecipeBase::VPReductionPHISC;
+ }
+ static inline bool classof(const VPValue *V) {
+ return V->getVPValueID() == VPValue::VPVReductionPHISC;
+ }
+ static inline bool classof(const VPWidenPHIRecipe *R) {
+ return R->getVPDefID() == VPRecipeBase::VPReductionPHISC;
+ }
+
+ /// Generate the phi/select nodes.
+ void execute(VPTransformState &State) override;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ /// Print the recipe.
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
+#endif
+
+ RecurrenceDescriptor &getRecurrenceDescriptor() { return RdxDesc; }
+
+ /// Returns true, if the phi is part of an ordered reduction.
+ bool isOrdered() const { return IsOrdered; }
+
+ /// Returns true, if the phi is part of an in-loop reduction.
+ bool isInLoop() const { return IsInLoop; }
};
/// A recipe for vectorizing a phi-node as a sequence of mask-based select
@@ -1257,10 +1309,6 @@ class VPReductionRecipe : public VPRecipeBase, public VPValue {
return V->getVPValueID() == VPValue::VPVReductionSC;
}
- static inline bool classof(const VPDef *D) {
- return D->getVPDefID() == VPRecipeBase::VPReductionSC;
- }
-
/// Generate the reduction in the loop
void execute(VPTransformState &State) override;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index e61caaacf2d3..cd0dd344f4c8 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -234,8 +234,7 @@ bool VPlanTransforms::mergeReplicateRegions(VPlan &Plan) {
auto *UI = dyn_cast<VPRecipeBase>(U);
if (!UI)
continue;
- auto *PhiR = dyn_cast<VPWidenPHIRecipe>(UI);
- if (PhiR && !PhiR->getRecurrenceDescriptor())
+ if (isa<VPWidenPHIRecipe>(UI) && !isa<VPReductionPHIRecipe>(UI))
return true;
}
return false;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 7b19e2f8a249..c4d05e38cbda 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -90,18 +90,22 @@ class VPValue {
/// type identification.
enum {
VPValueSC,
- VPVBlendSC,
VPVInstructionSC,
VPVMemoryInstructionSC,
- VPVPredInstPHI,
VPVReductionSC,
VPVReplicateSC,
VPVWidenSC,
VPVWidenCallSC,
VPVWidenGEPSC,
- VPVWidenIntOrFpIndcutionSC,
- VPVWidenPHISC,
VPVWidenSelectSC,
+
+ // Phi-like VPValues. Need to be kept together.
+ VPVBlendSC,
+ VPVWidenPHISC,
+ VPVWidenCanonicalIVSC,
+ VPVWidenIntOrFpInductionSC,
+ VPVPredInstPHI,
+ VPVReductionPHISC,
};
VPValue(Value *UV = nullptr, VPDef *Def = nullptr)
@@ -314,21 +318,26 @@ class VPDef {
/// SubclassID field of the VPRecipeBase objects. They are used for concrete
/// type identification.
using VPRecipeTy = enum {
- VPBlendSC,
VPBranchOnMaskSC,
VPInstructionSC,
VPInterleaveSC,
- VPPredInstPHISC,
VPReductionSC,
VPReplicateSC,
VPWidenCallSC,
- VPWidenCanonicalIVSC,
VPWidenGEPSC,
- VPWidenIntOrFpInductionSC,
VPWidenMemoryInstructionSC,
- VPWidenPHISC,
VPWidenSC,
- VPWidenSelectSC
+ VPWidenSelectSC,
+
+ // Phi-like recipes. Need to be kept together.
+ VPBlendSC,
+ VPWidenPHISC,
+ VPWidenCanonicalIVSC,
+ VPWidenIntOrFpInductionSC,
+ VPPredInstPHISC,
+ VPReductionPHISC,
+ VPFirstPHISC = VPBlendSC,
+ VPLastPHISC = VPReductionPHISC,
};
VPDef(const unsigned char SC) : SubclassID(SC) {}
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
index ed0cc712d2c5..8803a609da5d 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
@@ -143,7 +143,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, i32* %ptr) optsize
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
-; CHECK-NEXT: WIDEN-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next>
; CHECK-NEXT: EMIT vp<%4> = icmp ule ir<%iv> vp<%0>
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
index e4ce021b5f3b..def61c295945 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
@@ -79,7 +79,7 @@ define float @print_reduction(i64 %n, float* noalias %y) {
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
; CHECK-NEXT: for.body:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi %iv.next, 0
-; CHECK-NEXT: WIDEN-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next>
+; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%y>, ir<%iv>
; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx>
; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + reduce.fadd (ir<%lv>)
More information about the llvm-commits
mailing list