[llvm] [Mips] Fix wrong qNaN encoding when -mnan=legacy (PR #153777)

[via llvm-commits]

https://github.com/yingopq updated https://github.com/llvm/llvm-project/pull/153777

>From 31ad6fe67ffa0d7ac09cfe9fe07d4bdbb59bb017 Mon Sep 17 00:00:00 2001
From: Ying Huang <ying.huang at oss.cipunited.com>
Date: Fri, 15 Aug 2025 05:57:33 -0400
Subject: [PATCH] [Mips] Fix wrong qNaN encoding when -mnan=legacy

The correct value is:
Single floating-point 0x7fbf ffff(legacy) 0x7fc0 0000(nan2008)
Double floating-point 0x7ff7 ffff ffff ffff(legacy) 0x7ff8 0000 0000 0000(nan2008)

Fix #100495.
---
 .../CodeGen/SelectionDAG/TargetLowering.cpp   | 31 +++++++++++++++++--
 llvm/test/CodeGen/Mips/fminimum-fmaximum.ll   | 18 +++++++++++
 2 files changed, 47 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/Mips/fminimum-fmaximum.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 911bbabc42aa3..9758602685c00 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -8773,8 +8773,35 @@ SDValue TargetLowering::expandFMINIMUM_FMAXIMUM(SDNode *N,
   // Propagate any NaN of both operands
   if (!N->getFlags().hasNoNaNs() &&
       (!DAG.isKnownNeverNaN(RHS) || !DAG.isKnownNeverNaN(LHS))) {
-    ConstantFP *FPNaN = ConstantFP::get(*DAG.getContext(),
-                                        APFloat::getNaN(VT.getFltSemantics()));
+    ConstantFP *FPNaN = NULL;
+    bool hasNaN2008 = false;
+    if (DAG.getTarget().getTargetTriple().isMIPS()) {
+      const MCSubtargetInfo *STI = getTargetMachine().getMCSubtargetInfo();
+      StringRef FeatureString = STI->getFeatureString();
+      SubtargetFeatures Features(FeatureString);
+      for (const std::string &Feature : Features.getFeatures()) {
+        if (Feature == "+nan2008")
+          hasNaN2008 = true;
+      }
+      if (hasNaN2008 == false) {
+        if (&VT.getFltSemantics() == &APFloat::IEEEsingle()) {
+          APInt intPayload(64, 0xbfffff);
+          FPNaN = ConstantFP::get(
+              *DAG.getContext(),
+              APFloat::getSNaN(VT.getFltSemantics(), false, &intPayload));
+        } else if (&VT.getFltSemantics() == &APFloat::IEEEdouble()) {
+          APInt intPayload(64, 0x7ffffffffffff);
+          FPNaN = ConstantFP::get(
+              *DAG.getContext(),
+              APFloat::getSNaN(VT.getFltSemantics(), false, &intPayload));
+        }
+      } else
+        FPNaN = ConstantFP::get(*DAG.getContext(),
+                                APFloat::getNaN(VT.getFltSemantics()));
+    } else
+      FPNaN = ConstantFP::get(*DAG.getContext(),
+                              APFloat::getNaN(VT.getFltSemantics()));
+
     MinMax = DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, LHS, RHS, ISD::SETUO),
                            DAG.getConstantFP(*FPNaN, DL, VT), MinMax, Flags);
   }
diff --git a/llvm/test/CodeGen/Mips/fminimum-fmaximum.ll b/llvm/test/CodeGen/Mips/fminimum-fmaximum.ll
new file mode 100644
index 0000000000000..0121e8905303f
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/fminimum-fmaximum.ll
@@ -0,0 +1,18 @@
+; RUN: llc -mtriple=mips64el-unknown-linux-gnuabi64 -mcpu=mips64r2 -mattr=+nan2008 < %s | FileCheck -check-prefixes=NAN2008 %s
+; RUN: llc -mtriple=mips64el-unknown-linux-gnuabi64 -mcpu=mips64r2 -mattr=-nan2008 < %s | FileCheck -check-prefixes=LEGACY %s
+
+define float @test_fminimum_f32(float %a, float %b) {
+; NAN2008:    .4byte 0x7fc00000    # float NaN
+;
+; LEGACY:    .4byte 0x7fbfffff    # float NaN
+  %val = tail call float @llvm.minimum.f32(float %a, float %b)
+  ret float %val
+}
+
+define double @test_fminimum_f64(double %a, double %b) {
+; NAN2008:    .8byte 0x7ff8000000000000    # double NaN
+;
+; LEGACY:    .8byte 0x7ff7ffffffffffff    # double NaN
+  %val = tail call double @llvm.minimum.f64(double %a, double %b)
+  ret double %val
+}