[llvm] [VPlan] Try to hoist Previous (and operands), if sinking fails for FORs. (PR #108945)

[via llvm-commits]

================
@@ -771,6 +771,92 @@ sinkRecurrenceUsersAfterPrevious(VPFirstOrderRecurrencePHIRecipe *FOR,
   return true;
 }
 
+/// Try to hoist \p Previous and its operands before all users of \p FOR.
+static bool hoistPreviousBeforeFORUsers(VPFirstOrderRecurrencePHIRecipe *FOR,
+                                        VPRecipeBase *Previous,
+                                        VPDominatorTree &VPDT) {
+  using namespace llvm::VPlanPatternMatch;
+  if (Previous->mayHaveSideEffects() || Previous->mayReadFromMemory())
+    return false;
+
+  // Collect recipes that need hoisting.
+  SmallVector<VPRecipeBase *> WorkList;
+  SmallPtrSet<VPRecipeBase *, 8> Seen;
+  VPBasicBlock *HoistBlock = FOR->getParent();
+  auto HoistPoint = HoistBlock->getFirstNonPhi();
+  auto TryToPushHoistCandidate = [&](VPRecipeBase *HoistCandidate) {
+    // If we reach FOR, it means the original Previous depends on some other
+    // recurrence that in turn depends on FOR. If that is the case, we would
+    // also need to hoist recipes involving the other FOR, which may break
+    // dependencies.
+    if (HoistCandidate == FOR)
+      return false;
+
+    // Hoist candidate outside any region, no need to hoist.
+    if (!HoistCandidate->getParent()->getParent())
+      return true;
+
+    // Hoist candidate is a header phi or already visited, no need to hoist.
+    if (isa<VPHeaderPHIRecipe>(HoistCandidate) ||
+        !Seen.insert(HoistCandidate).second)
+      return true;
+
+    // If we reached a recipe that dominates all users of FOR, we don't need to
+    // hoist the recipe.
+    if (all_of(FOR->users(), [&VPDT, HoistCandidate](VPUser *U) {
+          return VPDT.properlyDominates(HoistCandidate, cast<VPRecipeBase>(U));
+        })) {
+      if (VPDT.properlyDominates(&*HoistPoint, HoistCandidate)) {
+        // This HoistCandidate domiantes all users of FOR and is closer to them
+        // than the previous HoistPoint.
+        HoistPoint = std::next(HoistCandidate->getIterator());
+        HoistBlock = HoistCandidate->getParent();
+      }
+      return true;
+    }
+
+    // Don't move candiates with sideeffects, as we do not yet analyze recipes
+    // between candidate and hoist destination yet.
+    if (HoistCandidate->mayHaveSideEffects())
+      return false;
+
+    WorkList.push_back(HoistCandidate);
+    return true;
+  };
+
+  // Recursively try to hoist Previous and its operands before all users of FOR.
+  // Update HoistPoint to the closest recipe that dominates all users of FOR.
+  if (!TryToPushHoistCandidate(Previous))
+    return false;
+
+  for (unsigned I = 0; I != WorkList.size(); ++I) {
+    VPRecipeBase *Current = WorkList[I];
+    assert(Current->getNumDefinedValues() == 1 &&
+           "only recipes with a single defined value expected");
+
+    for (VPValue *Op : Current->operands())
+      if (auto *R = Op->getDefiningRecipe())
----------------
ayalz wrote:

Live-ins could be considered along with header phis and out-of-loop recipes, as dominating users of FOR, but then Worklist will accommodate VPValues rather than recipes. Check (a) above for dominance could be applied to each operand before inserting into Worklist, as done here, rather than when retrieving it as described above. I.e., instead of a single `TryToPushHoistCandidate : recipe -> bool`, have two functions:
```
  SmallVector<VPRecipeBase *> HoistCandidates ({Previous});
  for (unsigned I = 0; I != HoistCandidates.size(); ++I) {
    VPRecipeBase *HoistCandidate = HoistCandidates[I];
    if (!CanHoistBefore(HoistCandidate))
      return false;
    for (VPValue *Op : HoistCandidate->operands())
      if (VPRecipeBase *OpCandidate = OpNeedsHoisting(Op))
        HoistCandidates.push_back(OpCandidate);
  }
```

https://github.com/llvm/llvm-project/pull/108945