[llvm] [SelectionDAG] Fix isKnownNeverNaN OR-logic for FMINNUM/FMAXNUM with SNaN (PR #204736)

[via llvm-commits]

https://github.com/ayrai-gb updated https://github.com/llvm/llvm-project/pull/204736

>From 2407462f84e7b63463552ea57c83e06cfff03475 Mon Sep 17 00:00:00 2001
From: Ayush Rai <your-email at amd.com>
Date: Mon, 29 Jun 2026 17:00:19 +0530
Subject: [PATCH] [SelectionDAG] Fix isKnownNeverNaN for FMINNUM/FMAXNUM with
 signaling NaNs

Co-authored-by: Cursor <cursoragent at cursor.com>
---
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 18 ++++++--
 .../fminmaxnum-snan-isknownnevernan.ll        | 44 +++++++++++++++++++
 2 files changed, 58 insertions(+), 4 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/fminmaxnum-snan-isknownnevernan.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index c542f8e7cc20b..a72885dcbdbbe 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -6423,10 +6423,20 @@ bool SelectionDAG::isKnownNeverNaN(SDValue Op, const APInt &DemandedElts,
   case ISD::FMAXNUM:
   case ISD::FMINIMUMNUM:
   case ISD::FMAXIMUMNUM: {
-    // Only one needs to be known not-nan, since it will be returned if the
-    // other ends up being one.
-    return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1) ||
-           isKnownNeverNaN(Op.getOperand(1), DemandedElts, SNaN, Depth + 1);
+    // The result is a NaN only if both operands are NaN, so it is never NaN
+    // (and hence never a signaling NaN) if either operand is known to never be
+    // NaN.
+    if (isKnownNeverNaN(Op.getOperand(0), DemandedElts, false, Depth + 1) ||
+        isKnownNeverNaN(Op.getOperand(1), DemandedElts, false, Depth + 1))
+      return true;
+    // Otherwise, for the signaling-NaN query, these do not quiet a signaling
+    // NaN: an input signaling NaN may be returned unchanged, so the result is
+    // never a signaling NaN only if both operands are known to never be one.
+    if (SNaN &&
+        isKnownNeverNaN(Op.getOperand(0), DemandedElts, true, Depth + 1) &&
+        isKnownNeverNaN(Op.getOperand(1), DemandedElts, true, Depth + 1))
+      return true;
+    return false;
   }
   case ISD::FMINNUM_IEEE:
   case ISD::FMAXNUM_IEEE: {
diff --git a/llvm/test/CodeGen/AArch64/fminmaxnum-snan-isknownnevernan.ll b/llvm/test/CodeGen/AArch64/fminmaxnum-snan-isknownnevernan.ll
new file mode 100644
index 0000000000000..1b904ce603c23
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fminmaxnum-snan-isknownnevernan.ll
@@ -0,0 +1,44 @@
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
+
+;; SelectionDAG::isKnownNeverNaN answers "is this value never a signaling NaN?"
+;; for FMINNUM/FMAXNUM/FMINIMUMNUM/FMAXIMUMNUM. These do not quiet a signaling
+;; NaN: when an operand is a signaling NaN it may be returned unchanged, so the
+;; result is known-never-signaling only if *both* operands are. The old code
+;; used OR (it sufficed for one operand to be known never-NaN), which is correct
+;; for the plain "never any NaN" query but unsound for the signaling query.
+;;
+;; This is observable through the bf16 narrowing in FP_TO_BF16 expansion, which
+;; inserts an FCANONICALIZE (quieting) step unless the source is known never to
+;; be a signaling NaN. On AArch64 that quieting lowers to setting the f32 quiet
+;; bit (orr #0x400000) under an unordered check (fcmp + csel vs).
+
+;; One operand is canonicalized (known never-sNaN), the other is unknown. The
+;; minnum result may be the unknown operand, so a signaling NaN can survive and
+;; the narrowing must quiet it. With the buggy OR-logic the source was wrongly
+;; proven never-sNaN and the quieting was dropped.
+; CHECK-LABEL: mixed:
+; CHECK:       fminnm s0, s0, s1
+; CHECK:       orr {{w[0-9]+}}, {{w[0-9]+}}, #0x400000
+; CHECK:       csel {{w[0-9]+}}, {{w[0-9]+}}, {{w[0-9]+}}, vs
+define bfloat @mixed(float %a, float %b) {
+  %na = call float @llvm.canonicalize.f32(float %a)
+  %m  = call float @llvm.minnum.f32(float %na, float %b)
+  %r  = fptrunc float %m to bfloat
+  ret bfloat %r
+}
+
+;; Both operands are canonicalized, so the minnum result really is never a
+;; signaling NaN and the narrowing may skip quieting. This guards against the
+;; fix becoming over-broad (the AND must still fold to true here).
+; CHECK-LABEL: both:
+; CHECK-NOT:   #0x400000
+define bfloat @both(float %a, float %b) {
+  %na = call float @llvm.canonicalize.f32(float %a)
+  %nb = call float @llvm.canonicalize.f32(float %b)
+  %m  = call float @llvm.minnum.f32(float %na, float %nb)
+  %r  = fptrunc float %m to bfloat
+  ret bfloat %r
+}
+
+declare float @llvm.canonicalize.f32(float)
+declare float @llvm.minnum.f32(float, float)