[PATCH] D100102: [VPlan] Use incoming VPValue to detect in-loop reductions (NFC).

[Florian Hahn via Phabricator via llvm-commits]

fhahn created this revision.
fhahn added reviewers: dmgreen, Ayal, gilr, rengolin.
Herald added subscribers: tschuett, psnobl, rogfer01, bollu, hiraditya.
fhahn requested review of this revision.
Herald added a subscriber: vkmr.
Herald added a project: LLVM.

After adding the incoming value from the backedge to the operands of
VPWidenPHIRecipe (D99294 <https://reviews.llvm.org/D99294>) , we can use that to check for in-loop
reductions.

This reduces the coupling with the cost model by using information
directly in VPlan.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D100102

Files:
  llvm/lib/Transforms/Vectorize/LoopVectorize.cpp


Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1584,11 +1584,6 @@
     return InLoopReductionChains;
   }
 
-  /// Returns true if the Phi is part of an inloop reduction.
-  bool isInLoopReduction(PHINode *Phi) const {
-    return InLoopReductionChains.count(Phi);
-  }
-
   /// Estimate cost of an intrinsic call instruction CI if it were vectorized
   /// with factor VF.  Return the cost of the instruction, including
   /// scalarization overhead if it's needed.
@@ -4283,7 +4278,8 @@
   TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue();
   Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();
   setDebugLocFromInst(Builder, ReductionStartValue);
-  bool IsInLoopReductionPhi = Cost->isInLoopReduction(OrigPhi);
+  bool IsInLoopReductionPhi = PhiR->getNumOperands() == 2 &&
+                              isa<VPReductionRecipe>(PhiR->getOperand(1));
 
   VPValue *LoopExitInstDef = State.Plan->getVPValue(LoopExitInst);
   // This is the vector-clone of the value that leaves the loop.
@@ -4693,7 +4689,8 @@
   if (RdxDesc || Legal->isFirstOrderRecurrence(P)) {
     Value *Iden = nullptr;
     bool ScalarPHI =
-        (State.VF.isScalar()) || Cost->isInLoopReduction(cast<PHINode>(PN));
+        (State.VF.isScalar()) || (PhiR->getNumOperands() == 2 &&
+                                  isa<VPReductionRecipe>(PhiR->getOperand(1)));
     Type *VecTy =
         ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), State.VF);
 
@@ -9039,11 +9036,15 @@
   if (CM.foldTailByMasking() && !Legal->getReductionVars().empty()) {
     Builder.setInsertPoint(VPBB);
     auto *Cond = RecipeBuilder.createBlockInMask(OrigLoop->getHeader(), Plan);
-    for (auto &Reduction : Legal->getReductionVars()) {
-      if (CM.isInLoopReduction(Reduction.first))
+    for (VPRecipeBase &R : Plan->getEntry()->getEntryBasicBlock()->phis()) {
+      VPWidenPHIRecipe *PhiR = dyn_cast<VPWidenPHIRecipe>(&R);
+      if (!PhiR || !PhiR->getRecurrenceDescriptor() ||
+          isa<VPReductionRecipe>(PhiR->getOperand(1)))
         continue;
-      VPValue *Phi = Plan->getOrAddVPValue(Reduction.first);
-      VPValue *Red = Plan->getOrAddVPValue(Reduction.second.getLoopExitInstr());
+
+      VPValue *Phi = PhiR;
+      VPValue *Red = Plan->getOrAddVPValue(
+          PhiR->getRecurrenceDescriptor()->getLoopExitInstr());
       Builder.createNaryOp(Instruction::Select, {Cond, Red, Phi});
     }
   }


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D100102.336082.patch
Type: text/x-patch
Size: 2608 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20210408/a8ba70cb/attachment.bin>