[llvm] [PowerPC] Fix IS_FPCLASS crash on 32-bit AIX targets for fcFinite/fcInf/fcInf|fcNan masks (PR #213298)

[zhijian lin via llvm-commits]

================
@@ -11991,7 +11992,79 @@ SDValue PPCTargetLowering::LowerIS_FPCLASS(SDValue Op,
   // - If value is not NaN, the comparison is equal (EQ bit set)
 
   if ((Category != fcNan) && (Category != ~fcNan)) {
-    // If not checking for NaN or non-NaN, we can't handle this without P9Vector
+    // On 32-bit PPC targets (where i64 is not a legal type), the generic
+    // integer-bitcast path in expandIS_FPCLASS produces illegal i64 nodes after
+    // type legalization has already run.  Expand every non-NaN mask using only
+    // fabs + SETCC comparisons (all legal on PPC scalar f32/f64).
+    //
+    // On 64-bit PPC we return SDValue() to let the generic expander use the
+    // integer-bitcast path which produces better code.
+    if (!Subtarget.isPPC64()) {
+      EVT ResultVT =
+          getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
+      const llvm::fltSemantics &Sem = VT == MVT::f64
+                                          ? llvm::APFloat::IEEEdouble()
+                                          : llvm::APFloat::IEEEsingle();
+      SDValue Abs = DAG.getNode(ISD::FABS, Dl, VT, LHS);
+      SDValue Inf = DAG.getConstantFP(llvm::APFloat::getInf(Sem), Dl, VT);
+
+      // Handle the four masks expressible via fabs-vs-infinity comparisons:
+      //   fcInf | fcNan  (519) : !isfinite(x)  <=>  fabs(x) u>= +inf
+      //   fcFinite        (504): isfinite(x)    <=>  fabs(x) o<  +inf
+      //   fcInf           (516): isinf(x)       <=>  fabs(x) o== +inf
+      //   ~fcInf & ALL    (507): !isinf(x)      <=>  fabs(x) u!= +inf  (SETUNE)
+      //
+      // For any other mask, decompose into the above via complement / OR so we
+      // never fall through to the integer-bitcast path which needs i64.
+      if (Category == (fcInf | fcNan)) {
+        // !isfinite(x) ==> fabs(x) u>= +inf
+        return DAG.getSetCC(Dl, ResultVT, Abs, Inf, ISD::SETUGE);
+      }
+      if (Category == fcFinite) {
+        // isfinite(x) ==> fabs(x) o< +inf
+        return DAG.getSetCC(Dl, ResultVT, Abs, Inf, ISD::SETOLT);
+      }
+      if (Category == fcInf) {
+        // isinf(x) ==> fabs(x) o== +inf
+        return DAG.getSetCC(Dl, ResultVT, Abs, Inf, ISD::SETOEQ);
+      }
+      // ~fcInf = all flags except fcPosInf and fcNegInf = "not inf"
+      // fabs(x) u!= +inf  (unordered-or-not-equal catches NaN too)
+      FPClassTest NotInf = static_cast<FPClassTest>(fcAllFlags & ~fcInf);
+      if (Category == NotInf)
+        return DAG.getSetCC(Dl, ResultVT, Abs, Inf, ISD::SETUNE);
+
+      // General case: decompose mask into (isinf | isnan) and/or isfinite
+      // parts and combine with OR/NOT.
+      //
+      // Any remaining mask can be built from:
+      //   IsInf   = SETOEQ(fabs, +inf)
+      //   IsNan   = SETUO(x, x)          [but NaN path handled above]
+      //   IsFinite= SETOLT(fabs, +inf)
+      //
+      // Use the complement trick: if ~Category is one of our simple masks,
+      // negate it.
+      FPClassTest InvCategory =
+          static_cast<FPClassTest>(fcAllFlags & ~Category);
+      SDValue InvResult;
+      if (InvCategory == (fcInf | fcNan))
+        InvResult = DAG.getSetCC(Dl, ResultVT, Abs, Inf, ISD::SETUGE);
+      else if (InvCategory == fcFinite)
+        InvResult = DAG.getSetCC(Dl, ResultVT, Abs, Inf, ISD::SETOLT);
+      else if (InvCategory == fcInf)
+        InvResult = DAG.getSetCC(Dl, ResultVT, Abs, Inf, ISD::SETOEQ);
+      else if (InvCategory == NotInf)
+        InvResult = DAG.getSetCC(Dl, ResultVT, Abs, Inf, ISD::SETUNE);
+      if (InvResult)
+        return DAG.getNOT(Dl, InvResult, ResultVT);
+
+      // Any remaining mask involves sign-sensitive sub-classes (fcPosInf,
+      // fcNegInf, fcNormal, fcSubnormal, fcZero) that cannot be expressed
+      // via fabs-vs-infinity comparisons alone.  Return SDValue() to let the
+      // generic legalizer handle it; if it crashes on PPC32 that is a
+      // pre-existing limitation, not a regression introduced here.
+      return SDValue();
----------------
diggerlin wrote:

I do not think we need the code line 12061~12066 ,  it will go to 12068 directly,

https://github.com/llvm/llvm-project/pull/213298