[llvm] 706bbfb - Revert "[VPlan] Add VPReductionPHIRecipe (NFC)." and follow-ups
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 6 04:10:33 PDT 2021
Author: Florian Hahn
Date: 2021-07-06T12:10:03+01:00
New Revision: 706bbfb35bd31051e46ac77aab3e9b2dbc3abe78
URL: https://github.com/llvm/llvm-project/commit/706bbfb35bd31051e46ac77aab3e9b2dbc3abe78
DIFF: https://github.com/llvm/llvm-project/commit/706bbfb35bd31051e46ac77aab3e9b2dbc3abe78.diff
LOG: Revert "[VPlan] Add VPReductionPHIRecipe (NFC)." and follow-ups
This reverts commit 3fed6d443f802c43aade1b5b1b09f5e2f8b3edb1,
bbcbf21ae60c928e07dde6a1c468763b3209d1e6 and
6c3451cd76cbd0cd973d9c2b08b168dcd0bce3c2.
The changes causing build failures with certain configurations, e.g.
https://lab.llvm.org/buildbot/#/builders/67/builds/3365/steps/6/logs/stdio
lib/libLLVMVectorize.a(LoopVectorize.cpp.o): In function `llvm::VPRecipeBuilder::tryToCreateWidenRecipe(llvm::Instruction*, llvm::ArrayRef<llvm::VPValue*>, llvm::VFRange&, std::unique_ptr<llvm::VPlan, std::default_delete<llvm::VPlan> >&) [clone .localalias.8]':
LoopVectorize.cpp:(.text._ZN4llvm15VPRecipeBuilder22tryToCreateWidenRecipeEPNS_11InstructionENS_8ArrayRefIPNS_7VPValueEEERNS_7VFRangeERSt10unique_ptrINS_5VPlanESt14default_deleteISA_EE+0x63b): undefined reference to `vtable for llvm::VPReductionPHIRecipe'
collect2: error: ld returned 1 exit status
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPlan.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanValue.h
llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
llvm/test/Transforms/LoopVectorize/vplan-printing.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 13b8216862e6..f3e5b020e8b6 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -503,11 +503,11 @@ class InnerLoopVectorizer {
unsigned UF, ElementCount VF, bool IsPtrLoopInvariant,
SmallBitVector &IsIndexLoopInvariant, VPTransformState &State);
- /// Vectorize a single first-order recurrence or pointer induction PHINode in
- /// a block. This method handles the induction variable canonicalization. It
- /// supports both VF = 1 for unrolled loops and arbitrary length vectors.
- void widenPHIInstruction(Instruction *PN, VPWidenPHIRecipe *PhiR,
- VPTransformState &State);
+ /// Vectorize a single PHINode in a block. This method handles the induction
+ /// variable canonicalization. It supports both VF = 1 for unrolled loops and
+ /// arbitrary length vectors.
+ void widenPHIInstruction(Instruction *PN, RecurrenceDescriptor *RdxDesc,
+ VPWidenPHIRecipe *PhiR, VPTransformState &State);
/// A helper function to scalarize a single Instruction in the innermost loop.
/// Generates a sequence of scalar instances for each lane between \p MinLane
@@ -596,7 +596,7 @@ class InnerLoopVectorizer {
/// Fix a reduction cross-iteration phi. This is the second phase of
/// vectorizing this phi node.
- void fixReduction(VPReductionPHIRecipe *Phi, VPTransformState &State);
+ void fixReduction(VPWidenPHIRecipe *Phi, VPTransformState &State);
/// Clear NSW/NUW flags from reduction instructions if necessary.
void clearReductionWrapFlags(const RecurrenceDescriptor &RdxDesc,
@@ -4135,8 +4135,8 @@ void InnerLoopVectorizer::fixCrossIterationPHIs(VPTransformState &State) {
if (!PhiR)
continue;
auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
- if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(PhiR)) {
- fixReduction(ReductionPhi, State);
+ if (PhiR->getRecurrenceDescriptor()) {
+ fixReduction(PhiR, State);
} else if (Legal->isFirstOrderRecurrence(OrigPhi))
fixFirstOrderRecurrence(PhiR, State);
}
@@ -4320,18 +4320,19 @@ void InnerLoopVectorizer::fixFirstOrderRecurrence(VPWidenPHIRecipe *PhiR,
LCSSAPhi.addIncoming(ExtractForPhiUsedOutsideLoop, LoopMiddleBlock);
}
-void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
+void InnerLoopVectorizer::fixReduction(VPWidenPHIRecipe *PhiR,
VPTransformState &State) {
PHINode *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
// Get it's reduction variable descriptor.
assert(Legal->isReductionVariable(OrigPhi) &&
"Unable to find the reduction variable");
- const RecurrenceDescriptor &RdxDesc = PhiR->getRecurrenceDescriptor();
+ const RecurrenceDescriptor &RdxDesc = *PhiR->getRecurrenceDescriptor();
RecurKind RK = RdxDesc.getRecurrenceKind();
TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue();
Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();
setDebugLocFromInst(ReductionStartValue);
+ bool IsInLoopReductionPhi = Cost->isInLoopReduction(OrigPhi);
VPValue *LoopExitInstDef = State.Plan->getVPValue(LoopExitInst);
// This is the vector-clone of the value that leaves the loop.
@@ -4346,11 +4347,14 @@ void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
// any loop invariant values.
BasicBlock *VectorLoopLatch = LI->getLoopFor(LoopVectorBody)->getLoopLatch();
- unsigned LastPartForNewPhi = PhiR->isOrdered() ? 1 : UF;
- for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
+ bool IsOrdered = IsInLoopReductionPhi && Cost->useOrderedReductions(RdxDesc);
+
+ for (unsigned Part = 0; Part < UF; ++Part) {
+ if (IsOrdered && Part > 0)
+ break;
Value *VecRdxPhi = State.get(PhiR->getVPSingleValue(), Part);
Value *Val = State.get(PhiR->getBackedgeValue(), Part);
- if (PhiR->isOrdered())
+ if (IsOrdered)
Val = State.get(PhiR->getBackedgeValue(), UF - 1);
cast<PHINode>(VecRdxPhi)->addIncoming(Val, VectorLoopLatch);
@@ -4369,7 +4373,7 @@ void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
// a Select choosing between the vectorized LoopExitInst and vectorized Phi,
// instead of the former. For an inloop reduction the reduction will already
// be predicated, and does not need to be handled here.
- if (Cost->foldTailByMasking() && !PhiR->isInLoop()) {
+ if (Cost->foldTailByMasking() && !IsInLoopReductionPhi) {
for (unsigned Part = 0; Part < UF; ++Part) {
Value *VecLoopExitInst = State.get(LoopExitInstDef, Part);
Value *Sel = nullptr;
@@ -4404,7 +4408,7 @@ void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
// then extend the loop exit value to enable InstCombine to evaluate the
// entire expression in the smaller type.
if (VF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) {
- assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
+ assert(!IsInLoopReductionPhi && "Unexpected truncated inloop reduction!");
Type *RdxVecTy = VectorType::get(RdxDesc.getRecurrenceType(), VF);
Builder.SetInsertPoint(
LI->getLoopFor(LoopVectorBody)->getLoopLatch()->getTerminator());
@@ -4442,7 +4446,7 @@ void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
// terminate on this line. This is the easiest way to ensure we don't
// accidentally cause an extra step back into the loop while debugging.
setDebugLocFromInst(LoopMiddleBlock->getTerminator());
- if (PhiR->isOrdered())
+ if (IsOrdered)
ReducedPartRdx = State.get(LoopExitInstDef, UF - 1);
else {
// Floating-point operations should have some FMF to enable the reduction.
@@ -4461,7 +4465,7 @@ void InnerLoopVectorizer::fixReduction(VPReductionPHIRecipe *PhiR,
// Create the reduction after the loop. Note that inloop reductions create the
// target reduction in the loop using a Reduction recipe.
- if (VF.isVector() && !PhiR->isInLoop()) {
+ if (VF.isVector() && !IsInLoopReductionPhi) {
ReducedPartRdx =
createTargetReduction(Builder, TTI, RdxDesc, ReducedPartRdx);
// If the reduction can be performed in a smaller type, we need to extend
@@ -4725,6 +4729,7 @@ void InnerLoopVectorizer::widenGEP(GetElementPtrInst *GEP, VPValue *VPDef,
}
void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN,
+ RecurrenceDescriptor *RdxDesc,
VPWidenPHIRecipe *PhiR,
VPTransformState &State) {
PHINode *P = cast<PHINode>(PN);
@@ -4750,21 +4755,68 @@ void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN,
// Phi nodes have cycles, so we need to vectorize them in two stages. This is
// stage #1: We create a new vector PHI node with no incoming edges. We'll use
// this value when we vectorize all of the instructions that use the PHI.
- if (Legal->isFirstOrderRecurrence(P)) {
- Type *VecTy = State.VF.isScalar()
- ? PN->getType()
- : VectorType::get(PN->getType(), State.VF);
-
- for (unsigned Part = 0; Part < State.UF; ++Part) {
+ if (RdxDesc || Legal->isFirstOrderRecurrence(P)) {
+ bool ScalarPHI =
+ (State.VF.isScalar()) || Cost->isInLoopReduction(cast<PHINode>(PN));
+ Type *VecTy =
+ ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), State.VF);
+
+ bool IsOrdered = Cost->isInLoopReduction(cast<PHINode>(PN)) &&
+ Cost->useOrderedReductions(*RdxDesc);
+ unsigned LastPartForNewPhi = IsOrdered ? 1 : State.UF;
+ for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
Value *EntryPart = PHINode::Create(
VecTy, 2, "vec.phi", &*LoopVectorBody->getFirstInsertionPt());
State.set(PhiR, EntryPart, Part);
}
+ if (Legal->isFirstOrderRecurrence(P))
return;
+ VPValue *StartVPV = PhiR->getStartValue();
+ Value *StartV = StartVPV->getLiveInIRValue();
+
+ Value *Iden = nullptr;
+
+ assert(Legal->isReductionVariable(P) && StartV &&
+ "RdxDesc should only be set for reduction variables; in that case "
+ "a StartV is also required");
+ RecurKind RK = RdxDesc->getRecurrenceKind();
+ if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) {
+ // MinMax reduction have the start value as their identify.
+ if (ScalarPHI) {
+ Iden = StartV;
+ } else {
+ IRBuilderBase::InsertPointGuard IPBuilder(Builder);
+ Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
+ StartV = Iden =
+ Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident");
+ }
+ } else {
+ Constant *IdenC = RecurrenceDescriptor::getRecurrenceIdentity(
+ RK, VecTy->getScalarType(), RdxDesc->getFastMathFlags());
+ Iden = IdenC;
+
+ if (!ScalarPHI) {
+ Iden = ConstantVector::getSplat(State.VF, IdenC);
+ IRBuilderBase::InsertPointGuard IPBuilder(Builder);
+ Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
+ Constant *Zero = Builder.getInt32(0);
+ StartV = Builder.CreateInsertElement(Iden, StartV, Zero);
+ }
+ }
+
+ for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
+ Value *EntryPart = State.get(PhiR, Part);
+ // Make sure to add the reduction start value only to the
+ // first unroll part.
+ Value *StartVal = (Part == 0) ? StartV : Iden;
+ cast<PHINode>(EntryPart)->addIncoming(StartVal, LoopVectorPreHeader);
+ }
+
+ return;
}
assert(!Legal->isReductionVariable(P) &&
- "reductions should be handled elsewhere");
+ "reductions should be handled above");
setDebugLocFromInst(P);
@@ -8926,9 +8978,7 @@ VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
RecurrenceDescriptor &RdxDesc = Legal->getReductionVars()[Phi];
assert(RdxDesc.getRecurrenceStartValue() ==
Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()));
- PhiRecipe = new VPReductionPHIRecipe(Phi, RdxDesc, *StartV,
- CM.isInLoopReduction(Phi),
- CM.useOrderedReductions(RdxDesc));
+ PhiRecipe = new VPWidenPHIRecipe(Phi, RdxDesc, *StartV);
} else {
PhiRecipe = new VPWidenPHIRecipe(Phi, *StartV);
}
@@ -9443,8 +9493,8 @@ void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
}
void VPWidenPHIRecipe::execute(VPTransformState &State) {
- State.ILV->widenPHIInstruction(cast<PHINode>(getUnderlyingValue()), this,
- State);
+ State.ILV->widenPHIInstruction(cast<PHINode>(getUnderlyingValue()), RdxDesc,
+ this, State);
}
void VPBlendRecipe::execute(VPTransformState &State) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index f8a41abddb5d..636b70272e44 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -763,7 +763,6 @@ void VPlan::execute(VPTransformState *State) {
State->VPValue2Value[Entry.second] = Entry.first;
BasicBlock *VectorPreHeaderBB = State->CFG.PrevBB;
- State->CFG.VectorPreHeader = VectorPreHeaderBB;
BasicBlock *VectorHeaderBB = VectorPreHeaderBB->getSingleSuccessor();
assert(VectorHeaderBB && "Loop preheader does not have a single successor.");
@@ -1115,74 +1114,6 @@ void VPWidenPHIRecipe::print(raw_ostream &O, const Twine &Indent,
printOperands(O, SlotTracker);
}
-void VPReductionPHIRecipe::print(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const {
- O << Indent << "WIDEN-REDUCTION-PHI ";
-
- printAsOperand(O, SlotTracker);
- O << " = phi ";
- printOperands(O, SlotTracker);
-}
-
-void VPReductionPHIRecipe::execute(VPTransformState &State) {
- PHINode *PN = cast<PHINode>(getUnderlyingValue());
- auto &Builder = State.Builder;
-
- // In order to support recurrences we need to be able to vectorize Phi nodes.
- // Phi nodes have cycles, so we need to vectorize them in two stages. This is
- // stage #1: We create a new vector PHI node with no incoming edges. We'll use
- // this value when we vectorize all of the instructions that use the PHI.
- bool ScalarPHI = State.VF.isScalar() || IsInLoop;
- Type *VecTy =
- ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), State.VF);
-
- BasicBlock *HeaderBB = State.CFG.PrevBB;
- assert(State.LI->getLoopFor(HeaderBB)->getHeader() == HeaderBB &&
- "recipe must be in the vector loop header");
- unsigned LastPartForNewPhi = isOrdered() ? 1 : State.UF;
- for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
- Value *EntryPart =
- PHINode::Create(VecTy, 2, "vec.phi", &*HeaderBB->getFirstInsertionPt());
- State.set(this, EntryPart, Part);
- }
- VPValue *StartVPV = getStartValue();
- Value *StartV = StartVPV->getLiveInIRValue();
-
- Value *Iden = nullptr;
- RecurKind RK = RdxDesc.getRecurrenceKind();
- if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) {
- // MinMax reduction have the start value as their identify.
- if (ScalarPHI) {
- Iden = StartV;
- } else {
- IRBuilderBase::InsertPointGuard IPBuilder(Builder);
- Builder.SetInsertPoint(State.CFG.VectorPreHeader->getTerminator());
- StartV = Iden =
- Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident");
- }
- } else {
- Constant *IdenC = RecurrenceDescriptor::getRecurrenceIdentity(
- RK, VecTy->getScalarType(), RdxDesc.getFastMathFlags());
- Iden = IdenC;
-
- if (!ScalarPHI) {
- Iden = ConstantVector::getSplat(State.VF, IdenC);
- IRBuilderBase::InsertPointGuard IPBuilder(Builder);
- Builder.SetInsertPoint(State.CFG.VectorPreHeader->getTerminator());
- Constant *Zero = Builder.getInt32(0);
- StartV = Builder.CreateInsertElement(Iden, StartV, Zero);
- }
- }
-
- for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
- Value *EntryPart = State.get(this, Part);
- // Make sure to add the reduction start value only to the
- // first unroll part.
- Value *StartVal = (Part == 0) ? StartV : Iden;
- cast<PHINode>(EntryPart)->addIncoming(StartVal, State.CFG.VectorPreHeader);
- }
-}
-
void VPBlendRecipe::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << Indent << "BLEND ";
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 83de2d47de64..b00c6156f9f6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -306,13 +306,6 @@ struct VPTransformState {
/// BasicBlock, used for placing the newly created BasicBlocks.
BasicBlock *LastBB = nullptr;
- /// The IR BasicBlock that is the preheader of the vector loop in the output
- /// IR.
- /// FIXME: The vector preheader should also be modeled in VPlan, so any code
- /// that needs to be added to the preheader gets directly generated by
- /// VPlan. There should be no need to manage a pointer to the IR BasicBlock.
- BasicBlock *VectorPreHeader = nullptr;
-
/// A mapping of each VPBasicBlock to the corresponding BasicBlock. In case
/// of replication, maps the BasicBlock of the last replica created.
SmallDenseMap<VPBasicBlock *, BasicBlock *> VPBB2IRBB;
@@ -737,7 +730,8 @@ class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
/// Returns true for PHI-like recipes.
bool isPhi() const {
- return getVPDefID() >= VPFirstPHISC && getVPDefID() <= VPLastPHISC;
+ return getVPDefID() == VPWidenIntOrFpInductionSC || getVPDefID() == VPWidenPHISC ||
+ getVPDefID() == VPPredInstPHISC || getVPDefID() == VPWidenCanonicalIVSC;
}
/// Returns true if the recipe may read from memory.
@@ -1050,48 +1044,54 @@ class VPWidenIntOrFpInductionRecipe : public VPRecipeBase {
}
};
-/// A recipe for handling first order recurrences and pointer inductions. For
-/// first-order recurrences, the start value is the first operand of the recipe
-/// and the incoming value from the backedge is the second operand. It also
-/// serves as base class for VPReductionPHIRecipe. In the VPlan native path, all
-/// incoming VPValues & VPBasicBlock pairs are managed in the recipe directly.
+/// A recipe for handling all phi nodes except for integer and FP inductions.
+/// For reduction PHIs, RdxDesc must point to the corresponding recurrence
+/// descriptor. For reductions and first-order recurrences, the start value is
+/// the first operand of the recipe and the incoming value from the backedge is
+/// the second operand. In the VPlan native path, all incoming VPValues &
+/// VPBasicBlock pairs are managed in the recipe directly.
class VPWidenPHIRecipe : public VPRecipeBase, public VPValue {
+ /// Descriptor for a reduction PHI.
+ RecurrenceDescriptor *RdxDesc = nullptr;
+
/// List of incoming blocks. Only used in the VPlan native path.
SmallVector<VPBasicBlock *, 2> IncomingBlocks;
-protected:
- VPWidenPHIRecipe(unsigned char VPVID, unsigned char VPDefID, PHINode *Phi)
- : VPRecipeBase(VPDefID, {}), VPValue(VPVID, Phi, this) {}
-
public:
- /// Create a VPWidenPHIRecipe for \p Phi
- VPWidenPHIRecipe(PHINode *Phi)
- : VPWidenPHIRecipe(VPVWidenPHISC, VPWidenPHISC, Phi) {}
+ /// Create a new VPWidenPHIRecipe for the reduction \p Phi described by \p
+ /// RdxDesc.
+ VPWidenPHIRecipe(PHINode *Phi, RecurrenceDescriptor &RdxDesc, VPValue &Start)
+ : VPWidenPHIRecipe(Phi) {
+ this->RdxDesc = &RdxDesc;
+ addOperand(&Start);
+ }
/// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start.
VPWidenPHIRecipe(PHINode *Phi, VPValue &Start) : VPWidenPHIRecipe(Phi) {
addOperand(&Start);
}
- virtual ~VPWidenPHIRecipe() override = default;
+ /// Create a VPWidenPHIRecipe for \p Phi
+ VPWidenPHIRecipe(PHINode *Phi)
+ : VPRecipeBase(VPWidenPHISC, {}),
+ VPValue(VPValue::VPVWidenPHISC, Phi, this) {}
+ ~VPWidenPHIRecipe() override = default;
/// Method to support type inquiry through isa, cast, and dyn_cast.
- static inline bool classof(const VPRecipeBase *B) {
- return B->getVPDefID() == VPRecipeBase::VPWidenPHISC ||
- B->getVPDefID() == VPRecipeBase::VPReductionPHISC;
+ static inline bool classof(const VPDef *D) {
+ return D->getVPDefID() == VPRecipeBase::VPWidenPHISC;
}
static inline bool classof(const VPValue *V) {
- return V->getVPValueID() == VPValue::VPVWidenPHISC ||
- V->getVPValueID() == VPValue::VPVReductionPHISC;
+ return V->getVPValueID() == VPValue::VPVWidenPHISC;
}
/// Generate the phi/select nodes.
- virtual void execute(VPTransformState &State) override;
+ void execute(VPTransformState &State) override;
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
/// Print the recipe.
- virtual void print(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const override;
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
#endif
/// Returns the start value of the phi, if it is a reduction or first-order
@@ -1117,62 +1117,8 @@ class VPWidenPHIRecipe : public VPRecipeBase, public VPValue {
/// Returns the \p I th incoming VPBasicBlock.
VPBasicBlock *getIncomingBlock(unsigned I) { return IncomingBlocks[I]; }
-};
-
-/// A recipe for handling reduction phis. The start value is the first operand
-/// of the recipe and the incoming value from the backedge is the second
-/// operand.
-class VPReductionPHIRecipe : public VPWidenPHIRecipe {
- /// Descriptor for the reduction.
- RecurrenceDescriptor &RdxDesc;
-
- /// The phi is part of an in-loop reduction.
- bool IsInLoop;
-
- /// The phi is part of an ordered reduction. Requires IsInLoop to be true.
- bool IsOrdered;
-
-public:
- /// Create a new VPReductionPHIRecipe for the reduction \p Phi described by \p
- /// RdxDesc.
- VPReductionPHIRecipe(PHINode *Phi, RecurrenceDescriptor &RdxDesc,
- VPValue &Start, bool IsInLoop = false,
- bool IsOrdered = false)
- : VPWidenPHIRecipe(VPVReductionPHISC, VPReductionPHISC, Phi),
- RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered) {
- assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop");
- addOperand(&Start);
- }
-
- ~VPReductionPHIRecipe() override = default;
-
- /// Method to support type inquiry through isa, cast, and dyn_cast.
- static inline bool classof(const VPRecipeBase *R) {
- return R->getVPDefID() == VPRecipeBase::VPReductionPHISC;
- }
- static inline bool classof(const VPValue *V) {
- return V->getVPValueID() == VPValue::VPVReductionPHISC;
- }
- static inline bool classof(const VPWidenPHIRecipe *R) {
- return R->getVPDefID() == VPRecipeBase::VPReductionPHISC;
- }
-
- /// Generate the phi/select nodes.
- void execute(VPTransformState &State) override;
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
- /// Print the recipe.
- void print(raw_ostream &O, const Twine &Indent,
- VPSlotTracker &SlotTracker) const override;
-#endif
-
- RecurrenceDescriptor &getRecurrenceDescriptor() { return RdxDesc; }
-
- /// Returns true, if the phi is part of an ordered reduction.
- bool isOrdered() const { return IsOrdered; }
-
- /// Returns true, if the phi is part of an in-loop reduction.
- bool isInLoop() const { return IsInLoop; }
+ RecurrenceDescriptor *getRecurrenceDescriptor() { return RdxDesc; }
};
/// A recipe for vectorizing a phi-node as a sequence of mask-based select
@@ -1311,6 +1257,10 @@ class VPReductionRecipe : public VPRecipeBase, public VPValue {
return V->getVPValueID() == VPValue::VPVReductionSC;
}
+ static inline bool classof(const VPDef *D) {
+ return D->getVPDefID() == VPRecipeBase::VPReductionSC;
+ }
+
/// Generate the reduction in the loop
void execute(VPTransformState &State) override;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index cd0dd344f4c8..e61caaacf2d3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -234,7 +234,8 @@ bool VPlanTransforms::mergeReplicateRegions(VPlan &Plan) {
auto *UI = dyn_cast<VPRecipeBase>(U);
if (!UI)
continue;
- if (isa<VPWidenPHIRecipe>(UI) && !isa<VPReductionPHIRecipe>(UI))
+ auto *PhiR = dyn_cast<VPWidenPHIRecipe>(UI);
+ if (PhiR && !PhiR->getRecurrenceDescriptor())
return true;
}
return false;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index c4d05e38cbda..7b19e2f8a249 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -90,22 +90,18 @@ class VPValue {
/// type identification.
enum {
VPValueSC,
+ VPVBlendSC,
VPVInstructionSC,
VPVMemoryInstructionSC,
+ VPVPredInstPHI,
VPVReductionSC,
VPVReplicateSC,
VPVWidenSC,
VPVWidenCallSC,
VPVWidenGEPSC,
- VPVWidenSelectSC,
-
- // Phi-like VPValues. Need to be kept together.
- VPVBlendSC,
+ VPVWidenIntOrFpIndcutionSC,
VPVWidenPHISC,
- VPVWidenCanonicalIVSC,
- VPVWidenIntOrFpInductionSC,
- VPVPredInstPHI,
- VPVReductionPHISC,
+ VPVWidenSelectSC,
};
VPValue(Value *UV = nullptr, VPDef *Def = nullptr)
@@ -318,26 +314,21 @@ class VPDef {
/// SubclassID field of the VPRecipeBase objects. They are used for concrete
/// type identification.
using VPRecipeTy = enum {
+ VPBlendSC,
VPBranchOnMaskSC,
VPInstructionSC,
VPInterleaveSC,
+ VPPredInstPHISC,
VPReductionSC,
VPReplicateSC,
VPWidenCallSC,
+ VPWidenCanonicalIVSC,
VPWidenGEPSC,
+ VPWidenIntOrFpInductionSC,
VPWidenMemoryInstructionSC,
- VPWidenSC,
- VPWidenSelectSC,
-
- // Phi-like recipes. Need to be kept together.
- VPBlendSC,
VPWidenPHISC,
- VPWidenCanonicalIVSC,
- VPWidenIntOrFpInductionSC,
- VPPredInstPHISC,
- VPReductionPHISC,
- VPFirstPHISC = VPBlendSC,
- VPLastPHISC = VPReductionPHISC,
+ VPWidenSC,
+ VPWidenSelectSC
};
VPDef(const unsigned char SC) : SubclassID(SC) {}
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
index 8803a609da5d..ed0cc712d2c5 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
@@ -143,7 +143,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, i32* %ptr) optsize
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next>
+; CHECK-NEXT: WIDEN-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next>
; CHECK-NEXT: EMIT vp<%4> = icmp ule ir<%iv> vp<%0>
; CHECK-NEXT: Successor(s): loop.0
; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
index def61c295945..e4ce021b5f3b 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll
@@ -79,7 +79,7 @@ define float @print_reduction(i64 %n, float* noalias %y) {
; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
; CHECK-NEXT: for.body:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi %iv.next, 0
-; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next>
+; CHECK-NEXT: WIDEN-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next>
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr ir<%y>, ir<%iv>
; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx>
; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + reduce.fadd (ir<%lv>)
More information about the llvm-commits
mailing list