[llvm] f1a6430 - [VPlan] Track both incoming values for first-order recurrence phis.
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Sun Jun 27 06:31:47 PDT 2021
Author: Florian Hahn
Date: 2021-06-27T14:29:35+01:00
New Revision: f1a64302725cc36e7796f327375b9e939e3a339e
URL: https://github.com/llvm/llvm-project/commit/f1a64302725cc36e7796f327375b9e939e3a339e
DIFF: https://github.com/llvm/llvm-project/commit/f1a64302725cc36e7796f327375b9e939e3a339e.diff
LOG: [VPlan] Track both incoming values for first-order recurrence phis.
This patch updates VPWidenPHI recipes for first-order recurrences to
also track the incoming value from the back-edge. Similar to D99294,
which did the same for reductions.
Reviewed By: Ayal
Differential Revision: https://reviews.llvm.org/D104197
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index b53c14424a700..e1dc71cd01801 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -591,7 +591,7 @@ class InnerLoopVectorizer {
/// Fix a first-order recurrence. This is the second phase of vectorizing
/// this phi node.
- void fixFirstOrderRecurrence(PHINode *Phi, VPTransformState &State);
+ void fixFirstOrderRecurrence(VPWidenPHIRecipe *PhiR, VPTransformState &State);
/// Fix a reduction cross-iteration phi. This is the second phase of
/// vectorizing this phi node.
@@ -4135,11 +4135,11 @@ void InnerLoopVectorizer::fixCrossIterationPHIs(VPTransformState &State) {
if (PhiR->getRecurrenceDescriptor()) {
fixReduction(PhiR, State);
} else if (Legal->isFirstOrderRecurrence(OrigPhi))
- fixFirstOrderRecurrence(OrigPhi, State);
+ fixFirstOrderRecurrence(PhiR, State);
}
}
-void InnerLoopVectorizer::fixFirstOrderRecurrence(PHINode *Phi,
+void InnerLoopVectorizer::fixFirstOrderRecurrence(VPWidenPHIRecipe *PhiR,
VPTransformState &State) {
// This is the second phase of vectorizing first-order recurrences. An
// overview of the transformation is described below. Suppose we have the
@@ -4190,13 +4190,7 @@ void InnerLoopVectorizer::fixFirstOrderRecurrence(PHINode *Phi,
// After execution completes the vector loop, we extract the next value of
// the recurrence (x) to use as the initial value in the scalar loop.
- // Get the original loop preheader and single loop latch.
- auto *Preheader = OrigLoop->getLoopPreheader();
- auto *Latch = OrigLoop->getLoopLatch();
-
- // Get the initial and previous values of the scalar recurrence.
- auto *ScalarInit = Phi->getIncomingValueForBlock(Preheader);
- auto *Previous = Phi->getIncomingValueForBlock(Latch);
+ auto *ScalarInit = PhiR->getStartValue()->getLiveInIRValue();
auto *IdxTy = Builder.getInt32Ty();
auto *One = ConstantInt::get(IdxTy, 1);
@@ -4212,11 +4206,10 @@ void InnerLoopVectorizer::fixFirstOrderRecurrence(PHINode *Phi,
VectorInit, LastIdx, "vector.recur.init");
}
- VPValue *PhiDef = State.Plan->getVPValue(Phi);
- VPValue *PreviousDef = State.Plan->getVPValue(Previous);
+ VPValue *PreviousDef = PhiR->getBackedgeValue();
// We constructed a temporary phi node in the first phase of vectorization.
// This phi node will eventually be deleted.
- Builder.SetInsertPoint(cast<Instruction>(State.get(PhiDef, 0)));
+ Builder.SetInsertPoint(cast<Instruction>(State.get(PhiR, 0)));
// Create a phi node for the new recurrence. The current value will either be
// the initial value inserted into a vector or loop-varying vector value.
@@ -4256,13 +4249,13 @@ void InnerLoopVectorizer::fixFirstOrderRecurrence(PHINode *Phi,
// Shuffle the current and previous vector and update the vector parts.
for (unsigned Part = 0; Part < UF; ++Part) {
Value *PreviousPart = State.get(PreviousDef, Part);
- Value *PhiPart = State.get(PhiDef, Part);
+ Value *PhiPart = State.get(PhiR, Part);
auto *Shuffle = VF.isVector()
? Builder.CreateVectorSplice(Incoming, PreviousPart, -1)
: Incoming;
PhiPart->replaceAllUsesWith(Shuffle);
cast<Instruction>(PhiPart)->eraseFromParent();
- State.reset(PhiDef, Shuffle, Part);
+ State.reset(PhiR, Shuffle, Part);
Incoming = PreviousPart;
}
@@ -4299,6 +4292,7 @@ void InnerLoopVectorizer::fixFirstOrderRecurrence(PHINode *Phi,
// Fix the initial value of the original recurrence in the scalar loop.
Builder.SetInsertPoint(&*LoopScalarPreHeader->begin());
+ PHINode *Phi = cast<PHINode>(PhiR->getUnderlyingValue());
auto *Start = Builder.CreatePHI(Phi->getType(), 2, "scalar.recur.init");
for (auto *BB : predecessors(LoopScalarPreHeader)) {
auto *Incoming = BB == LoopMiddleBlock ? ExtractForScalar : ScalarInit;
@@ -4755,49 +4749,16 @@ void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN,
assert(PN->getParent() == OrigLoop->getHeader() &&
"Non-header phis should have been handled elsewhere");
- VPValue *StartVPV = PhiR->getStartValue();
- Value *StartV = StartVPV ? StartVPV->getLiveInIRValue() : nullptr;
// In order to support recurrences we need to be able to vectorize Phi nodes.
// Phi nodes have cycles, so we need to vectorize them in two stages. This is
// stage #1: We create a new vector PHI node with no incoming edges. We'll use
// this value when we vectorize all of the instructions that use the PHI.
if (RdxDesc || Legal->isFirstOrderRecurrence(P)) {
- Value *Iden = nullptr;
bool ScalarPHI =
(State.VF.isScalar()) || Cost->isInLoopReduction(cast<PHINode>(PN));
Type *VecTy =
ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), State.VF);
- if (RdxDesc) {
- assert(Legal->isReductionVariable(P) && StartV &&
- "RdxDesc should only be set for reduction variables; in that case "
- "a StartV is also required");
- RecurKind RK = RdxDesc->getRecurrenceKind();
- if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) {
- // MinMax reduction have the start value as their identify.
- if (ScalarPHI) {
- Iden = StartV;
- } else {
- IRBuilderBase::InsertPointGuard IPBuilder(Builder);
- Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
- StartV = Iden =
- Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident");
- }
- } else {
- Constant *IdenC = RecurrenceDescriptor::getRecurrenceIdentity(
- RK, VecTy->getScalarType(), RdxDesc->getFastMathFlags());
- Iden = IdenC;
-
- if (!ScalarPHI) {
- Iden = ConstantVector::getSplat(State.VF, IdenC);
- IRBuilderBase::InsertPointGuard IPBuilder(Builder);
- Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
- Constant *Zero = Builder.getInt32(0);
- StartV = Builder.CreateInsertElement(Iden, StartV, Zero);
- }
- }
- }
-
bool IsOrdered = State.VF.isVector() &&
Cost->isInLoopReduction(cast<PHINode>(PN)) &&
Cost->useOrderedReductions(*RdxDesc);
@@ -4806,13 +4767,50 @@ void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN,
Value *EntryPart = PHINode::Create(
VecTy, 2, "vec.phi", &*LoopVectorBody->getFirstInsertionPt());
State.set(PhiR, EntryPart, Part);
- if (StartV) {
- // Make sure to add the reduction start value only to the
- // first unroll part.
- Value *StartVal = (Part == 0) ? StartV : Iden;
- cast<PHINode>(EntryPart)->addIncoming(StartVal, LoopVectorPreHeader);
+ }
+ if (Legal->isFirstOrderRecurrence(P))
+ return;
+ VPValue *StartVPV = PhiR->getStartValue();
+ Value *StartV = StartVPV->getLiveInIRValue();
+
+ Value *Iden = nullptr;
+
+ assert(Legal->isReductionVariable(P) && StartV &&
+ "RdxDesc should only be set for reduction variables; in that case "
+ "a StartV is also required");
+ RecurKind RK = RdxDesc->getRecurrenceKind();
+ if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK)) {
+ // MinMax reduction have the start value as their identify.
+ if (ScalarPHI) {
+ Iden = StartV;
+ } else {
+ IRBuilderBase::InsertPointGuard IPBuilder(Builder);
+ Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
+ StartV = Iden =
+ Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident");
+ }
+ } else {
+ Constant *IdenC = RecurrenceDescriptor::getRecurrenceIdentity(
+ RK, VecTy->getScalarType(), RdxDesc->getFastMathFlags());
+ Iden = IdenC;
+
+ if (!ScalarPHI) {
+ Iden = ConstantVector::getSplat(State.VF, IdenC);
+ IRBuilderBase::InsertPointGuard IPBuilder(Builder);
+ Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
+ Constant *Zero = Builder.getInt32(0);
+ StartV = Builder.CreateInsertElement(Iden, StartV, Zero);
}
}
+
+ for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
+ Value *EntryPart = State.get(PhiR, Part);
+ // Make sure to add the reduction start value only to the
+ // first unroll part.
+ Value *StartVal = (Part == 0) ? StartV : Iden;
+ cast<PHINode>(EntryPart)->addIncoming(StartVal, LoopVectorPreHeader);
+ }
+
return;
}
@@ -8958,22 +8956,32 @@ VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands)))
return toVPRecipeResult(Recipe);
- if (Legal->isReductionVariable(Phi)) {
- RecurrenceDescriptor &RdxDesc = Legal->getReductionVars()[Phi];
- assert(RdxDesc.getRecurrenceStartValue() ==
- Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()));
+ VPWidenPHIRecipe *PhiRecipe = nullptr;
+ if (Legal->isReductionVariable(Phi) || Legal->isFirstOrderRecurrence(Phi)) {
VPValue *StartV = Operands[0];
+ if (Legal->isReductionVariable(Phi)) {
+ RecurrenceDescriptor &RdxDesc = Legal->getReductionVars()[Phi];
+ assert(RdxDesc.getRecurrenceStartValue() ==
+ Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()));
+ PhiRecipe = new VPWidenPHIRecipe(Phi, RdxDesc, *StartV);
+ } else {
+ PhiRecipe = new VPWidenPHIRecipe(Phi, *StartV);
+ }
- auto *PhiRecipe = new VPWidenPHIRecipe(Phi, RdxDesc, *StartV);
- PhisToFix.push_back(PhiRecipe);
// Record the incoming value from the backedge, so we can add the incoming
// value from the backedge after all recipes have been created.
recordRecipeOf(cast<Instruction>(
Phi->getIncomingValueForBlock(OrigLoop->getLoopLatch())));
- return toVPRecipeResult(PhiRecipe);
+ PhisToFix.push_back(PhiRecipe);
+ } else {
+ // TODO: record start and backedge value for remaining pointer induction
+ // phis.
+ assert(Phi->getType()->isPointerTy() &&
+ "only pointer phis should be handled here");
+ PhiRecipe = new VPWidenPHIRecipe(Phi);
}
- return toVPRecipeResult(new VPWidenPHIRecipe(Phi));
+ return toVPRecipeResult(PhiRecipe);
}
if (isa<TruncInst>(Instr) &&
diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
index bc1a039acb430..65857f0342104 100644
--- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -56,8 +56,9 @@ class VPRecipeBuilder {
// marked by having a nullptr entry in this map.
DenseMap<Instruction *, VPRecipeBase *> Ingredient2Recipe;
- /// Cross-iteration reduction phis for which we need to add the incoming value
- /// from the backedge after all recipes have been created.
+ /// Cross-iteration reduction & first-order recurrence phis for which we need
+ /// to add the incoming value from the backedge after all recipes have been
+ /// created.
SmallVector<VPWidenPHIRecipe *, 4> PhisToFix;
/// Check if \p I can be widened at the start of \p Range and possibly
@@ -170,8 +171,8 @@ class VPRecipeBuilder {
Instruction *I, VFRange &Range, VPBasicBlock *VPBB,
VPlanPtr &Plan);
- /// Add the incoming values from the backedge to reduction cross-iteration
- /// phis.
+ /// Add the incoming values from the backedge to reduction & first-order
+ /// recurrence cross-iteration phis.
void fixHeaderPhis();
};
} // end namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index a9fd99ff2a6a9..b00c6156f9f6b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1046,10 +1046,10 @@ class VPWidenIntOrFpInductionRecipe : public VPRecipeBase {
/// A recipe for handling all phi nodes except for integer and FP inductions.
/// For reduction PHIs, RdxDesc must point to the corresponding recurrence
-/// descriptor, the start value is the first operand of the recipe and the
-/// incoming value from the backedge is the second operand. In the VPlan native
-/// path, all incoming VPValues & VPBasicBlock pairs are managed in the recipe
-/// directly.
+/// descriptor. For reductions and first-order recurrences, the start value is
+/// the first operand of the recipe and the incoming value from the backedge is
+/// the second operand. In the VPlan native path, all incoming VPValues &
+/// VPBasicBlock pairs are managed in the recipe directly.
class VPWidenPHIRecipe : public VPRecipeBase, public VPValue {
/// Descriptor for a reduction PHI.
RecurrenceDescriptor *RdxDesc = nullptr;
@@ -1066,6 +1066,11 @@ class VPWidenPHIRecipe : public VPRecipeBase, public VPValue {
addOperand(&Start);
}
+ /// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start.
+ VPWidenPHIRecipe(PHINode *Phi, VPValue &Start) : VPWidenPHIRecipe(Phi) {
+ addOperand(&Start);
+ }
+
/// Create a VPWidenPHIRecipe for \p Phi
VPWidenPHIRecipe(PHINode *Phi)
: VPRecipeBase(VPWidenPHISC, {}),
@@ -1089,15 +1094,15 @@ class VPWidenPHIRecipe : public VPRecipeBase, public VPValue {
VPSlotTracker &SlotTracker) const override;
#endif
- /// Returns the start value of the phi, if it is a reduction.
+ /// Returns the start value of the phi, if it is a reduction or first-order
+ /// recurrence.
VPValue *getStartValue() {
return getNumOperands() == 0 ? nullptr : getOperand(0);
}
- /// Returns the incoming value from the loop backedge, if it is a reduction.
+ /// Returns the incoming value from the loop backedge, if it is a reduction or
+ /// first-order recurrence.
VPValue *getBackedgeValue() {
- assert(RdxDesc && "second incoming value is only guaranteed to be backedge "
- "value for reductions");
return getOperand(1);
}
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
index fc1422fb4bad3..d38fc97226a48 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
@@ -10,7 +10,7 @@ define void @sink_replicate_region_1(i32 %x, i8* %ptr) optsize {
; CHECK-LABEL: sink_replicate_region_1
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: loop:
-; CHECK-NEXT: WIDEN-PHI %0 = phi 0, %conv
+; CHECK-NEXT: WIDEN-PHI ir<%0> = phi ir<0>, ir<%conv>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: EMIT vp<%3> = icmp ule ir<%iv> vp<%0>
; CHECK-NEXT: Successor(s): loop.0
@@ -83,7 +83,7 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, i32* %ptr) optsize {
; CHECK-LABEL: sink_replicate_region_2
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: loop:
-; CHECK-NEXT: WIDEN-PHI %recur = phi 0, %recur.next
+; CHECK-NEXT: WIDEN-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: EMIT vp<%3> = icmp ule ir<%iv> vp<%0>
; CHECK-NEXT: Successor(s): loop.0
@@ -155,7 +155,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, i32* %ptr) optsize
; CHECK-LABEL: sink_replicate_region_3_reduction
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: loop:
-; CHECK-NEXT: WIDEN-PHI %recur = phi 0, %recur.next
+; CHECK-NEXT: WIDEN-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: WIDEN-PHI ir<%and.red> = phi ir<1234>, ir<%and.red.next>
; CHECK-NEXT: EMIT vp<%4> = icmp ule ir<%iv> vp<%0>
@@ -214,7 +214,7 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, i8*
; CHECK-LABEL: sink_replicate_region_4_requires_split_at_end_of_block
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: loop:
-; CHECK-NEXT: WIDEN-PHI %0 = phi 0, %conv
+; CHECK-NEXT: WIDEN-PHI ir<%0> = phi ir<0>, ir<%conv>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: EMIT vp<%3> = icmp ule ir<%iv> vp<%0>
; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, ir<%iv>
@@ -312,7 +312,7 @@ define void @sink_replicate_region_after_replicate_region(i32* %ptr, i32 %x, i8
; CHECK-LABEL: sink_replicate_region_after_replicate_region
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: loop:
-; CHECK-NEXT: WIDEN-PHI %recur = phi 0, %recur.next
+; CHECK-NEXT: WIDEN-PHI ir<%recur> = phi ir<0>, ir<%recur.next>
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
; CHECK-NEXT: EMIT vp<%3> = icmp ule ir<%iv> vp<%0>
; CHECK-NEXT: Successor(s): loop.0
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
index 48eecbe04f122..c7156fd3f70dc 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
@@ -849,7 +849,7 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' {
; CHECK-NEXT: loop:
; CHECK-NEXT: WIDEN-INDUCTION %iv = phi 0, %iv.next
-; CHECK-NEXT: WIDEN-PHI %for = phi 0, %lv.a
+; CHECK-NEXT: WIDEN-PHI ir<%for> = phi ir<0>, ir<%lv.a>
; CHECK-NEXT: EMIT vp<%3> = icmp ule ir<%iv> vp<%0>
; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%iv>
; CHECK-NEXT: Successor(s): pred.load
More information about the llvm-commits
mailing list