[PATCH] D100102: [VPlan] Use incoming VPValue to detect in-loop reductions (NFC).

[Florian Hahn via Phabricator via llvm-commits]

fhahn updated this revision to Diff 340869.
fhahn added a comment.

Remove unnecessary check on number of operands, add assertion to catch if we miss an in-loop reduction in VPlan.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D100102/new/

https://reviews.llvm.org/D100102

Files:
  llvm/lib/Transforms/Vectorize/LoopVectorize.cpp


Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1584,11 +1584,6 @@
     return InLoopReductionChains;
   }
 
-  /// Returns true if the Phi is part of an inloop reduction.
-  bool isInLoopReduction(PHINode *Phi) const {
-    return InLoopReductionChains.count(Phi);
-  }
-
   /// Estimate cost of an intrinsic call instruction CI if it were vectorized
   /// with factor VF.  Return the cost of the instruction, including
   /// scalarization overhead if it's needed.
@@ -4281,7 +4276,7 @@
   TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue();
   Instruction *LoopExitInst = RdxDesc.getLoopExitInstr();
   setDebugLocFromInst(Builder, ReductionStartValue);
-  bool IsInLoopReductionPhi = Cost->isInLoopReduction(OrigPhi);
+  bool IsInLoopReductionPhi = isa<VPReductionRecipe>(PhiR->getBackedgeValue());
 
   VPValue *LoopExitInstDef = State.Plan->getVPValue(LoopExitInst);
   // This is the vector-clone of the value that leaves the loop.
@@ -4692,7 +4687,8 @@
   if (RdxDesc || Legal->isFirstOrderRecurrence(P)) {
     Value *Iden = nullptr;
     bool ScalarPHI =
-        (State.VF.isScalar()) || Cost->isInLoopReduction(cast<PHINode>(PN));
+        (State.VF.isScalar()) ||
+        (RdxDesc && isa<VPReductionRecipe>(PhiR->getBackedgeValue()));
     Type *VecTy =
         ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), State.VF);
 
@@ -9053,16 +9049,29 @@
 
   // Finally, if tail is folded by masking, introduce selects between the phi
   // and the live-out instruction of each reduction, at the end of the latch.
-  if (CM.foldTailByMasking() && !Legal->getReductionVars().empty()) {
-    Builder.setInsertPoint(VPBB);
-    auto *Cond = RecipeBuilder.createBlockInMask(OrigLoop->getHeader(), Plan);
-    for (auto &Reduction : Legal->getReductionVars()) {
-      if (CM.isInLoopReduction(Reduction.first))
+  if (CM.foldTailByMasking()) {
+    VPValue *Cond = nullptr;
+    unsigned NumRegularReductions = 0;
+    for (VPRecipeBase &R : Plan->getEntry()->getEntryBasicBlock()->phis()) {
+      VPWidenPHIRecipe *PhiR = dyn_cast<VPWidenPHIRecipe>(&R);
+      if (!PhiR || !PhiR->getRecurrenceDescriptor() ||
+          isa<VPReductionRecipe>(PhiR->getBackedgeValue()))
         continue;
-      VPValue *Phi = Plan->getOrAddVPValue(Reduction.first);
-      VPValue *Red = Plan->getOrAddVPValue(Reduction.second.getLoopExitInstr());
+
+      if (!Cond) {
+        Builder.setInsertPoint(VPBB);
+        Cond = RecipeBuilder.createBlockInMask(OrigLoop->getHeader(), Plan);
+      }
+      VPValue *Phi = PhiR;
+      VPValue *Red = Plan->getOrAddVPValue(
+          PhiR->getRecurrenceDescriptor()->getLoopExitInstr());
       Builder.createNaryOp(Instruction::Select, {Cond, Red, Phi});
+      NumRegularReductions++;
     }
+    assert(!Range.Start.isVector() ||
+           NumRegularReductions == Legal->getReductionVars().size() -
+                                       CM.getInLoopReductionChains().size() &&
+               "failed to identify an in-loop reduction in VPlan");
   }
 
   std::string PlanName;


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D100102.340869.patch
Type: text/x-patch
Size: 3241 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20210427/893d8799/attachment.bin>