[llvm] [IVDescriptors] Identify min/max recurrences in single pass. (PR #163460)

[Florian Hahn via llvm-commits]

================
@@ -214,6 +214,173 @@ static bool checkOrderedReduction(RecurKind Kind, Instruction *ExactFPMathInst,
   return true;
 }
 
+// Helper to collect FMF from a value and its associated fcmp in select patterns
+static FastMathFlags collectMinMaxFMF(Value *V) {
+  FastMathFlags FMF = cast<FPMathOperator>(V)->getFastMathFlags();
+  if (auto *Sel = dyn_cast<SelectInst>(V)) {
+    // Accept FMF on either fcmp or select of a min/max idiom.
+    // TODO: This is a hack to work-around the fact that FMF may not be
+    //       assigned/propagated correctly. If that problem is fixed or we
+    //       standardize on fmin/fmax via intrinsics, this can be removed.
+    if (auto *FCmp = dyn_cast<FCmpInst>(Sel->getCondition()))
+      FMF |= FCmp->getFastMathFlags();
+  }
+  return FMF;
+}
+
+static std::optional<FastMathFlags>
+hasRequiredFastMathFlags(FPMathOperator *FPOp, RecurKind &RK,
+                         FastMathFlags FuncFMF) {
+  bool HasRequiredFMF =
+      (FuncFMF.noNaNs() && FuncFMF.noSignedZeros()) ||
+      (FPOp && FPOp->hasNoNaNs() && FPOp->hasNoSignedZeros()) ||
+      RK == RecurKind::FMinimum || RK == RecurKind::FMaximum ||
+      RK == RecurKind::FMinimumNum || RK == RecurKind::FMaximumNum;
+  if (!HasRequiredFMF) {
+    if (RK == RecurKind::FMax &&
+        match(FPOp, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value())))
+      RK = RecurKind::FMaxNum;
+    else if (RK == RecurKind::FMin &&
+             match(FPOp, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value())))
+      RK = RecurKind::FMinNum;
+    else
+      return std::nullopt;
+  }
+  return {collectMinMaxFMF(FPOp)};
+}
+
+static std::optional<RecurrenceDescriptor>
+getMultiUseMinMax(PHINode *Phi, Loop *TheLoop, FastMathFlags FuncFMF,
+                  ScalarEvolution *SE) {
+  if (Phi->getNumIncomingValues() != 2 ||
+      Phi->getParent() != TheLoop->getHeader())
+    return std::nullopt;
+
+  Type *Ty = Phi->getType();
+  BasicBlock *Latch = TheLoop->getLoopLatch();
+  if ((!Ty->isIntegerTy() && !Ty->isFloatingPointTy()) || !Latch)
+    return std::nullopt;
+
+  auto Matches = [](Value *V, Value *&A, Value *&B) -> RecurKind {
+    if (match(V, m_UMin(m_Value(A), m_Value(B))))
+      return RecurKind::UMin;
+    if (match(V, m_UMax(m_Value(A), m_Value(B))))
+      return RecurKind::UMax;
+    if (match(V, m_SMax(m_Value(A), m_Value(B))))
+      return RecurKind::SMax;
+    if (match(V, m_SMin(m_Value(A), m_Value(B))))
+      return RecurKind::SMin;
+    if (match(V, m_OrdOrUnordFMin(m_Value(A), m_Value(B))) ||
+        match(V, m_Intrinsic<Intrinsic::minnum>(m_Value(A), m_Value(B))))
+      return RecurKind::FMin;
+    if (match(V, m_OrdOrUnordFMax(m_Value(A), m_Value(B))) ||
+        match(V, m_Intrinsic<Intrinsic::maxnum>(m_Value(A), m_Value(B))))
+      return RecurKind::FMax;
+    if (match(V, m_FMinimum(m_Value(A), m_Value(B))))
+      return RecurKind::FMinimum;
+    if (match(V, m_FMaximum(m_Value(A), m_Value(B))))
+      return RecurKind::FMaximum;
+    if (match(V, m_Intrinsic<Intrinsic::minimumnum>(m_Value(A), m_Value(B))))
+      return RecurKind::FMinimumNum;
+    if (match(V, m_Intrinsic<Intrinsic::maximumnum>(m_Value(A), m_Value(B))))
+      return RecurKind::FMaximumNum;
+    return RecurKind::None;
+  };
+
+  FastMathFlags FMF = FastMathFlags::getFast();
+  Value *RdxNext = Phi->getIncomingValueForBlock(Latch);
+  RecurKind RK = RecurKind::None;
+  // Identify min/max recurrences by walking the def-use chains upwards,
+  // starting at RdxNext.
+  SmallVector<Value *> WorkList = {RdxNext};
+  SmallPtrSet<Value *, 8> Chain = {Phi};
+  while (!WorkList.empty()) {
+    Value *Cur = WorkList.pop_back_val();
+    if (!Chain.insert(Cur).second)
+      continue;
+    auto *I = dyn_cast<Instruction>(Cur);
+    if (!I || !TheLoop->contains(I))
+      return std::nullopt;
+    if (auto *PN = dyn_cast<PHINode>(I)) {
+      if (PN != Phi)
+        append_range(WorkList, PN->operands());
+      continue;
+    }
+    Value *A, *B;
+    RecurKind CurRK = Matches(Cur, A, B);
+    if (CurRK == RecurKind::None || (RK != RecurKind::None && CurRK != RK))
+      return std::nullopt;
+
+    RK = CurRK;
+    // For floating point recurrences, check we have the required fast-math
+    // flags.
+    if (RecurrenceDescriptor::isFPMinMaxRecurrenceKind(CurRK)) {
+      if (auto CurFMF =
+              hasRequiredFastMathFlags(cast<FPMathOperator>(Cur), RK, FuncFMF))
+        FMF &= *CurFMF;
+      else
+        return std::nullopt;
+    }
+
+    Chain.insert(I);
+    if (auto *SI = dyn_cast<SelectInst>(I))
+      Chain.insert(SI->getCondition());
+
+    if (A == Phi || B == Phi)
+      continue;
+
+    // Add operand to worklist if it matches the pattern - exactly one must
+    // match
+    Value *X, *Y;
+    auto *IA = dyn_cast<Instruction>(A);
+    auto *IB = dyn_cast<Instruction>(B);
+    bool AMatches = IA && TheLoop->contains(IA) && Matches(A, X, Y) == RK;
+    bool BMatches = IB && TheLoop->contains(IB) && Matches(B, X, Y) == RK;
+    if (AMatches == BMatches) // Both or neither match
+      return std::nullopt;
+    WorkList.push_back(AMatches ? A : B);
+  }
+
+  // Check users of RdxNext. It can have
+  // * a single user outside the loop,
+  // * used stores to the same invariant address,
+  // * used by the starting recurrence phis.
+  unsigned IncOut = 0;
+  StoreInst *IntermediateStore = nullptr;
+  for (Use &U : RdxNext->uses()) {
+    auto *User = cast<Instruction>(U.getUser());
+    if (!TheLoop->contains(User->getParent())) {
+      if (++IncOut > 1)
----------------
fhahn wrote:

updated thanks

https://github.com/llvm/llvm-project/pull/163460