[llvm] [LV] Vectorize maxnum/minnum w/o fast-math flags. (PR #148239)

[Florian Hahn via llvm-commits]

================
@@ -628,3 +628,140 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
     Term->addMetadata(LLVMContext::MD_prof, BranchWeights);
   }
 }
+
+static VPValue *getMinMaxCompareValue(VPSingleDefRecipe *MinMaxOp,
+                                      VPReductionPHIRecipe *RedPhi) {
+  auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxOp);
+  if (!isa<VPWidenIntrinsicRecipe>(MinMaxOp) &&
+      !(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
+    return nullptr;
+
+  if (MinMaxOp->getOperand(0) == RedPhi)
+    return MinMaxOp->getOperand(1);
+  return MinMaxOp->getOperand(0);
+}
+
+/// Returns true if there VPlan is read-only and execution can be resumed at the
+/// beginning of the last vector iteration in the scalar loop
+static bool canResumeInScalarLoopFromVectorLoop(VPlan &Plan) {
+  for (VPBlockBase *VPB : vp_depth_first_shallow(
+           Plan.getVectorLoopRegion()->getEntryBasicBlock())) {
+    auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
+    if (!VPBB)
+      return false;
+    for (auto &R : *VPBB) {
+      if (match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
+        continue;
+      if (R.mayWriteToMemory())
+        return false;
+    }
+  }
+  return true;
+}
+
+bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
+  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+  VPValue *AnyNaN = nullptr;
+  VPReductionPHIRecipe *RedPhiR = nullptr;
+  VPRecipeWithIRFlags *MinMaxOp = nullptr;
+  bool HasUnsupportedPhi = false;
+  for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
+    HasUnsupportedPhi |=
+        !isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
+             VPReductionPHIRecipe>(&R);
+    auto *Cur = dyn_cast<VPReductionPHIRecipe>(&R);
+    if (!Cur)
+      continue;
+    if (RedPhiR)
+      return false;
+    if (Cur->getRecurrenceKind() != RecurKind::FMaxNumNoFMFs &&
+        Cur->getRecurrenceKind() != RecurKind::FMinNumNoFMFs)
+      continue;
+
+    RedPhiR = Cur;
+    MinMaxOp = dyn_cast<VPRecipeWithIRFlags>(
+        RedPhiR->getBackedgeValue()->getDefiningRecipe());
+    if (!MinMaxOp)
+      return false;
+    VPValue *In = getMinMaxCompareValue(MinMaxOp, RedPhiR);
+    if (!In)
+      return false;
+
+    auto *IsNaN =
+        new VPInstruction(Instruction::FCmp, {In, In}, {CmpInst::FCMP_UNO}, {});
+    IsNaN->insertBefore(MinMaxOp);
+    AnyNaN = new VPInstruction(VPInstruction::AnyOf, {IsNaN});
+    AnyNaN->getDefiningRecipe()->insertAfter(IsNaN);
+  }
+
+  if (!AnyNaN)
+    return true;
+
+  if (HasUnsupportedPhi || !canResumeInScalarLoopFromVectorLoop(Plan))
+    return false;
+
+  auto *MiddleVPBB = Plan.getMiddleBlock();
+  auto *RdxResult = dyn_cast<VPInstruction>(&MiddleVPBB->front());
+  if (!RdxResult ||
+      RdxResult->getOpcode() != VPInstruction::ComputeReductionResult ||
+      RdxResult->getOperand(0) != RedPhiR)
----------------
fhahn wrote:

Updated find via the uses of the reduction phi, thanks

https://github.com/llvm/llvm-project/pull/148239