[PATCH] D157039: [VPlan] Add VPlan::hasTailFolded.

[Florian Hahn via Phabricator via llvm-commits]

fhahn created this revision.
fhahn added reviewers: Ayal, gilr, dmgreen.
Herald added subscribers: StephenFan, tschuett, psnobl, rogfer01, bollu, hiraditya.
Herald added a project: All.
fhahn requested review of this revision.
Herald added subscribers: wangpc, vkmr.
Herald added a project: LLVM.

This patch adds a helper to check if the tail loop is folded in a
VPlan.

To do so it checks if the VPlan contains a widen canonical IV, a
active-lane-mask-phi, an active-lane-mask or a ICmpULE compare of a
widened canonical induction with the backedge taken count. This should
effectively check if a mask for the header is present, but may be a bit
cumbersome.

Alternatively we could also add a flag to the VPlan.

Depends on D157037 <https://reviews.llvm.org/D157037>.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D157039

Files:
  llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
  llvm/lib/Transforms/Vectorize/VPlan.cpp
  llvm/lib/Transforms/Vectorize/VPlan.h


Index: llvm/lib/Transforms/Vectorize/VPlan.h
===================================================================
--- llvm/lib/Transforms/Vectorize/VPlan.h
+++ llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2653,6 +2653,9 @@
   VPBasicBlock *getPreheader() { return Preheader; }
   const VPBasicBlock *getPreheader() const { return Preheader; }
 
+  /// Returns true if the tail loop is folded in the VPlan.
+  bool hasTailFolded();
+
 private:
   /// Add to the given dominator tree the header block and every new basic block
   /// that was created between it and the latch block, inclusive.
Index: llvm/lib/Transforms/Vectorize/VPlan.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -845,6 +845,32 @@
   LiveOuts.insert({PN, new VPLiveOut(PN, V)});
 }
 
+bool VPlan::hasTailFolded() {
+  VPBasicBlock *HeaderVPBB = getVectorLoopRegion()->getEntryBasicBlock();
+  return any_of(*HeaderVPBB, [this](VPRecipeBase &R) {
+    // A widened canonical IV or active-lane-mask PHI imply the tail being
+    // folded.
+    if (isa<VPWidenCanonicalIVRecipe>(&R) || isa<VPActiveLaneMaskPHIRecipe>(&R))
+      return true;
+
+    auto *VPI = dyn_cast<VPInstruction>(&R);
+    if (!VPI)
+      return false;
+    if (VPI->getOpcode() == VPInstruction::ActiveLaneMask)
+      return true;
+    if (VPI->getOpcode() != VPInstruction::ICmpULE)
+      return false;
+    auto IsWidenCanonicalIV = [](const VPRecipeBase &R) {
+      if (auto *Ind = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R))
+        return Ind->isCanonical();
+      return isa<VPWidenCanonicalIVRecipe>(&R);
+    };
+    if (IsWidenCanonicalIV(*VPI->getOperand(0)->getDefiningRecipe()) &&
+        VPI->getOperand(1) == getOrCreateBackedgeTakenCount())
+      return true;
+  });
+}
+
 void VPlan::updateDominatorTree(DominatorTree *DT, BasicBlock *LoopHeaderBB,
                                 BasicBlock *LoopLatchBB,
                                 BasicBlock *LoopExitBB) {
Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3914,7 +3914,7 @@
   // a Select choosing between the vectorized LoopExitInst and vectorized Phi,
   // instead of the former. For an inloop reduction the reduction will already
   // be predicated, and does not need to be handled here.
-  if (Cost->foldTailByMasking() && !PhiR->isInLoop()) {
+  if (PhiR->getParent()->getPlan()->hasTailFolded() && !PhiR->isInLoop()) {
     for (unsigned Part = 0; Part < UF; ++Part) {
       Value *VecLoopExitInst = State.get(LoopExitInstDef, Part);
       SelectInst *Sel = nullptr;
@@ -9256,7 +9256,7 @@
   // If tail is folded by masking, introduce selects between the phi
   // and the live-out instruction of each reduction, at the beginning of the
   // dedicated latch block.
-  if (CM.foldTailByMasking()) {
+  if (Plan->hasTailFolded()) {
     Builder.setInsertPoint(LatchVPBB, LatchVPBB->begin());
     for (VPRecipeBase &R :
          Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D157039.547007.patch
Type: text/x-patch
Size: 3218 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20230803/463a9391/attachment.bin>