[PATCH] D99294: [LV] Track incoming values for reductions in recipe (NFC).

[Florian Hahn via Phabricator via llvm-commits]

fhahn updated this revision to Diff 338872.
fhahn added a comment.

Update comment to clarify that the first operand of a VPWidenPHIRecipe for a reduction is the start value and the second operand is the incoming value from the backedge.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D99294/new/

https://reviews.llvm.org/D99294

Files:
  llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
  llvm/lib/Transforms/Vectorize/VPlan.h
  llvm/test/Transforms/LoopVectorize/vplan-printing.ll


Index: llvm/test/Transforms/LoopVectorize/vplan-printing.ll
===================================================================
--- llvm/test/Transforms/LoopVectorize/vplan-printing.ll
+++ llvm/test/Transforms/LoopVectorize/vplan-printing.ll
@@ -79,7 +79,7 @@
 ; CHECK:      VPlan 'Initial VPlan for VF={4},UF>=1' {
 ; CHECK-NEXT: for.body:
 ; CHECK-NEXT:   WIDEN-INDUCTION %iv = phi %iv.next, 0
-; CHECK-NEXT:   WIDEN-PHI %red = phi %red.next, 0.000000e+00
+; CHECK-NEXT:   WIDEN-PHI ir<%red> = phi ir<0.000000e+00>, ir<%red.next>
 ; CHECK-NEXT:   CLONE ir<%arrayidx> = getelementptr ir<%y>, ir<%iv>
 ; CHECK-NEXT:   WIDEN ir<%lv> = load ir<%arrayidx>
 ; CHECK-NEXT:   REDUCE ir<%red.next> = ir<%red> + reduce.fadd (ir<%lv>)
Index: llvm/lib/Transforms/Vectorize/VPlan.h
===================================================================
--- llvm/lib/Transforms/Vectorize/VPlan.h
+++ llvm/lib/Transforms/Vectorize/VPlan.h
@@ -987,9 +987,10 @@
 
 /// A recipe for handling all phi nodes except for integer and FP inductions.
 /// For reduction PHIs, RdxDesc must point to the corresponding recurrence
-/// descriptor and the start value is the first operand of the recipe.
-/// In the VPlan native path, all incoming VPValues & VPBasicBlock pairs are
-/// managed in the recipe directly.
+/// descriptor, the start value is the first operand of the recipe and the
+/// incoming value from the backedge is the second operand. In the VPlan native
+/// path, all incoming VPValues & VPBasicBlock pairs are managed in the recipe
+/// directly.
 class VPWidenPHIRecipe : public VPRecipeBase, public VPValue {
   /// Descriptor for a reduction PHI.
   RecurrenceDescriptor *RdxDesc = nullptr;
Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -4297,17 +4297,15 @@
 
   // Reductions do not have to start at zero. They can start with
   // any loop invariant values.
-  BasicBlock *Latch = OrigLoop->getLoopLatch();
-  Value *LoopVal = OrigPhi->getIncomingValueForBlock(Latch);
-
   for (unsigned Part = 0; Part < UF; ++Part) {
-    Value *VecRdxPhi = State.get(PhiR->getVPValue(), Part);
-    Value *Val = State.get(State.Plan->getVPValue(LoopVal), Part);
+    Value *VecRdxPhi = State.get(PhiR->getVPValue(0), Part);
+    Value *Val = State.get(PhiR->getOperand(1), Part);
     if (IsInLoopReductionPhi && useOrderedReductions(RdxDesc) &&
         State.VF.isVector())
-      Val = State.get(State.Plan->getVPValue(LoopVal), UF - 1);
-    cast<PHINode>(VecRdxPhi)
-      ->addIncoming(Val, LI->getLoopFor(LoopVectorBody)->getLoopLatch());
+      Val = State.get(PhiR->getOperand(1), UF - 1);
+
+    cast<PHINode>(VecRdxPhi)->addIncoming(
+        Val, LI->getLoopFor(LoopVectorBody)->getLoopLatch());
   }
 
   // Before each round, move the insertion point right between
@@ -8863,6 +8861,18 @@
     RecipeBuilder.recordRecipeOf(Entry.first);
     RecipeBuilder.recordRecipeOf(Entry.second);
   }
+
+  SmallVector<PHINode *, 4> PhisToFix;
+  BasicBlock *OrigLatch = OrigLoop->getLoopLatch();
+  for (PHINode &Phi : OrigLoop->getHeader()->phis()) {
+    if (!Legal->isReductionVariable(&Phi))
+      continue;
+    PhisToFix.push_back(&Phi);
+    RecipeBuilder.recordRecipeOf(&Phi);
+    RecipeBuilder.recordRecipeOf(
+        cast<Instruction>(Phi.getIncomingValueForBlock(OrigLatch)));
+  }
+
   for (auto &Reduction : CM.getInLoopReductionChains()) {
     PHINode *Phi = Reduction.first;
     RecurKind Kind = Legal->getReductionVars()[Phi].getRecurrenceKind();
@@ -8970,6 +8980,14 @@
     }
   }
 
+  for (PHINode *PN : PhisToFix) {
+    VPWidenPHIRecipe *R = cast<VPWidenPHIRecipe>(RecipeBuilder.getRecipe(PN));
+    VPRecipeBase *IncR = RecipeBuilder.getRecipe(
+        cast<Instruction>(PN->getIncomingValueForBlock(OrigLatch)));
+    assert(IncR->getNumDefinedValues() == 1);
+    R->addOperand(IncR->getVPValue(0));
+  }
+
   // Discard empty dummy pre-entry VPBasicBlock. Note that other VPBasicBlocks
   // may also be empty, such as the last one VPBB, reflecting original
   // basic-blocks with no recipes.


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D99294.338872.patch
Type: text/x-patch
Size: 4195 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20210420/be6e0aed/attachment.bin>