[llvm] [VPlan] Try to hoist Previous (and operands), if sinking fails for FORs. (PR #108945)

[Florian Hahn via llvm-commits]

================
@@ -771,6 +771,92 @@ sinkRecurrenceUsersAfterPrevious(VPFirstOrderRecurrencePHIRecipe *FOR,
   return true;
 }
 
+/// Try to hoist \p Previous and its operands before all users of \p FOR.
+static bool hoistPreviousBeforeFORUsers(VPFirstOrderRecurrencePHIRecipe *FOR,
+                                        VPRecipeBase *Previous,
+                                        VPDominatorTree &VPDT) {
+  using namespace llvm::VPlanPatternMatch;
+  if (Previous->mayHaveSideEffects() || Previous->mayReadFromMemory())
+    return false;
+
+  // Collect recipes that need hoisting.
+  SmallVector<VPRecipeBase *> WorkList;
+  SmallPtrSet<VPRecipeBase *, 8> Seen;
+  VPBasicBlock *HoistBlock = FOR->getParent();
+  auto HoistPoint = HoistBlock->getFirstNonPhi();
+  auto TryToPushHoistCandidate = [&](VPRecipeBase *HoistCandidate) {
+    // If we reach FOR, it means the original Previous depends on some other
+    // recurrence that in turn depends on FOR. If that is the case, we would
+    // also need to hoist recipes involving the other FOR, which may break
+    // dependencies.
+    if (HoistCandidate == FOR)
+      return false;
+
+    // Hoist candidate outside any region, no need to hoist.
+    if (!HoistCandidate->getParent()->getParent())
+      return true;
+
+    // Hoist candidate is a header phi or already visited, no need to hoist.
+    if (isa<VPHeaderPHIRecipe>(HoistCandidate) ||
+        !Seen.insert(HoistCandidate).second)
+      return true;
+
+    // If we reached a recipe that dominates all users of FOR, we don't need to
+    // hoist the recipe.
+    if (all_of(FOR->users(), [&VPDT, HoistCandidate](VPUser *U) {
+          return VPDT.properlyDominates(HoistCandidate, cast<VPRecipeBase>(U));
+        })) {
+      if (VPDT.properlyDominates(&*HoistPoint, HoistCandidate)) {
+        // This HoistCandidate domiantes all users of FOR and is closer to them
+        // than the previous HoistPoint.
+        HoistPoint = std::next(HoistCandidate->getIterator());
+        HoistBlock = HoistCandidate->getParent();
+      }
+      return true;
+    }
+
+    // Don't move candiates with sideeffects, as we do not yet analyze recipes
+    // between candidate and hoist destination yet.
+    if (HoistCandidate->mayHaveSideEffects())
+      return false;
+
+    WorkList.push_back(HoistCandidate);
+    return true;
+  };
+
+  // Recursively try to hoist Previous and its operands before all users of FOR.
+  // Update HoistPoint to the closest recipe that dominates all users of FOR.
+  if (!TryToPushHoistCandidate(Previous))
----------------
fhahn wrote:

Updated, thanks!

https://github.com/llvm/llvm-project/pull/108945