[libclc] [libclc] Fix fmod returning NaN for subnormal inputs (PR #225351)

[via cfe-commits]

https://github.com/Lurie97 created https://github.com/llvm/llvm-project/pull/225351

#222369 classifies the NaN/Inf/zero edge cases with __clc_isnan and __clc_isfinite, which lower to __builtin_isfpclass. Some targets misclassify subnormals there when denormals are supported, so a subnormal x with a normal y (|x| < |y|, which must return x unchanged) yields NaN:

  fmod(-0x1.26e13p-128, -0x1.2afc64p+16) -> nan, expected -0x1.26e13p-128

Classify with plain bit arithmetic as the pre-#222369 code did. The reduction loop is unchanged.

Fixes OpenCL-CTS math_brute_force fmod on a device advertising CL_FP_DENORM (i.MX95 Mali-G310, Panfrost/Rusticl).

>From e9873f20ca939f3d6a04f06f2051ebe6b7d1f5af Mon Sep 17 00:00:00 2001
From: jiajia Qian <jiajia.qian at nxp.com>
Date: Wed, 23 Sep 2026 17:14:31 +0800
Subject: [PATCH] [libclc] Fix fmod returning NaN for subnormal inputs

#222369 classifies the NaN/Inf/zero edge cases with __clc_isnan and
__clc_isfinite, which lower to __builtin_isfpclass. Some targets
misclassify subnormals there when denormals are supported, so a subnormal
x with a normal y (|x| < |y|, which must return x unchanged) yields NaN:

  fmod(-0x1.26e13p-128, -0x1.2afc64p+16) -> nan, expected -0x1.26e13p-128

Classify with plain bit arithmetic as the pre-#222369 code did. The
reduction loop is unchanged.

Fixes OpenCL-CTS math_brute_force fmod on a device advertising
CL_FP_DENORM (i.MX95 Mali-G310, Panfrost/Rusticl).

Signed-off-by: jiajia Qian <jiajia.qian at nxp.com>
---
 libclc/clc/lib/generic/math/clc_fmod.cl | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/libclc/clc/lib/generic/math/clc_fmod.cl b/libclc/clc/lib/generic/math/clc_fmod.cl
index 699d95485b71e9..64d52e40a042a7 100644
--- a/libclc/clc/lib/generic/math/clc_fmod.cl
+++ b/libclc/clc/lib/generic/math/clc_fmod.cl
@@ -17,8 +17,6 @@
 #include <clc/math/clc_recip_fast.h>
 #include <clc/math/clc_rint.h>
 #include <clc/math/math.h>
-#include <clc/relational/clc_isfinite.h>
-#include <clc/relational/clc_isnan.h>
 
 _CLC_DEF _CLC_OVERLOAD float __clc_fmod(float x, float y) {
   // How many bits of the quotient to resolve per iteration.
@@ -69,9 +67,15 @@ _CLC_DEF _CLC_OVERLOAD float __clc_fmod(float x, float y) {
   }
 
   // fmod(x, 0) is NaN; fmod(Inf, y) is NaN; fmod(x, NaN)/fmod(NaN, y) is NaN.
-  ret = y == 0.0f ? FLT_NAN : ret;
-  int c = !__clc_isnan(y) && __clc_isfinite(x);
-  ret = c ? ret : FLT_NAN;
+  // Classify with plain bit arithmetic rather than __clc_isnan/__clc_isfinite:
+  // those lower to __builtin_isfpclass, which some targets implement in a way
+  // that misclassifies subnormal inputs when denormals are supported, turning
+  // the |x| < |y| result (a subnormal x) into a NaN.
+  int axbits = __clc_as_int(x) & EXSIGNBIT_SP32;
+  int aybits = __clc_as_int(y) & EXSIGNBIT_SP32;
+  int c = axbits > PINFBITPATT_SP32 | aybits > PINFBITPATT_SP32 |
+          axbits == PINFBITPATT_SP32 | aybits == 0;
+  ret = c ? __clc_as_float(QNANBITPATT_SP32) : ret;
 
   return ret;
 }
@@ -136,9 +140,12 @@ _CLC_DEF _CLC_OVERLOAD double __clc_fmod(double x, double y) {
   }
 
   // fmod(x, 0) is NaN; fmod(Inf, y) is NaN; fmod(x, NaN)/fmod(NaN, y) is NaN.
-  ret = y == 0.0 ? DBL_NAN : ret;
-  int c = !__clc_isnan(y) && __clc_isfinite(x);
-  ret = c ? ret : DBL_NAN;
+  // See the FP32 version for why this avoids __clc_isnan/__clc_isfinite.
+  ulong axbits = __clc_as_ulong(x) & ~SIGNBIT_DP64;
+  ulong aybits = __clc_as_ulong(y) & ~SIGNBIT_DP64;
+  int c = axbits > PINFBITPATT_DP64 || aybits > PINFBITPATT_DP64 ||
+          axbits == PINFBITPATT_DP64 || aybits == 0;
+  ret = c ? __clc_as_double(QNANBITPATT_DP64) : ret;
 
   return ret;
 }