[llvm] [SimplifyLibCalls] Don't set nnan on synthesized frem in optimizeFMod (PR #199284)

[Justin Lebar via llvm-commits]

https://github.com/jlebar updated https://github.com/llvm/llvm-project/pull/199284

>From e2de399b17f4cafa5994ab9699bb00f821aafaba Mon Sep 17 00:00:00 2001
From: Justin Lebar <justin.lebar at gmail.com>
Date: Fri, 22 May 2026 00:22:25 -0700
Subject: [PATCH] [SimplifyLibCalls] Don't set nnan on synthesized frem in
 optimizeFMod

LibCallSimplifier can fold a libcall to fmod to an frem instruction if
the fmod call doesn't set errno.

fmod(x, y) sets errno if x == +/-Inf or y == 0.  The old code assumed
that this was also a sufficient condition to prove that the result is
not NaN, and so unconditionally set the nnan fmf on the new frem
instruction.  That's not sound; e.g. fmod(x, NaN) == frem(x, NaN) ==
NaN.

We don't actually have to worry about propagating the `nnan` flag;
B.CreateFRemFMF does it for us automagically.

This bug was found by a large run of Opus 4.7 looking for bugs in LLVM.
---
 .../lib/Transforms/Utils/SimplifyLibCalls.cpp | 19 ++++++---------
 llvm/test/Transforms/InstCombine/fmod.ll      | 24 +++++++++++++++----
 2 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index b60f0c7021d51..9584e0e1c825f 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -2853,11 +2853,10 @@ Value *LibCallSimplifier::optimizeSqrt(CallInst *CI, IRBuilderBase &B) {
 
 Value *LibCallSimplifier::optimizeFMod(CallInst *CI, IRBuilderBase &B) {
 
-  // fmod(x,y) can set errno if y == 0 or x == +/-inf, and returns Nan in those
-  // case. If we know those do not happen, then we can convert the fmod into
-  // frem.
-  bool IsNoNan = CI->hasNoNaNs();
-  if (!IsNoNan) {
+  // fmod(x,y) sets errno if y == 0 or x == +/-inf. frem does not set errno,
+  // so the fold is valid only when we can prove fmod wouldn't either.
+  bool IsNoErrno = CI->hasNoNaNs();
+  if (!IsNoErrno) {
     SimplifyQuery SQ(DL, TLI, DT, AC, CI, true, true, DC);
     KnownFPClass Known0 = computeKnownFPClass(CI->getOperand(0), fcInf, SQ);
     if (Known0.isKnownNeverInfinity()) {
@@ -2866,16 +2865,12 @@ Value *LibCallSimplifier::optimizeFMod(CallInst *CI, IRBuilderBase &B) {
       Function *F = CI->getParent()->getParent();
       const fltSemantics &FltSem =
           CI->getType()->getScalarType()->getFltSemantics();
-      IsNoNan = Known1.isKnownNeverLogicalZero(F->getDenormalMode(FltSem));
+      IsNoErrno = Known1.isKnownNeverLogicalZero(F->getDenormalMode(FltSem));
     }
   }
 
-  if (IsNoNan) {
-    Value *FRem = B.CreateFRemFMF(CI->getOperand(0), CI->getOperand(1), CI);
-    if (auto *FRemI = dyn_cast<Instruction>(FRem))
-      FRemI->setHasNoNaNs(true);
-    return FRem;
-  }
+  if (IsNoErrno)
+    return B.CreateFRemFMF(CI->getOperand(0), CI->getOperand(1), CI);
   return nullptr;
 }
 
diff --git a/llvm/test/Transforms/InstCombine/fmod.ll b/llvm/test/Transforms/InstCombine/fmod.ll
index 804812abadbbb..a1ef7d6b2c61a 100644
--- a/llvm/test/Transforms/InstCombine/fmod.ll
+++ b/llvm/test/Transforms/InstCombine/fmod.ll
@@ -9,7 +9,7 @@ define float @test_inf_const(float %f) {
 ; CHECK-NEXT:    [[ISINF:%.*]] = fcmp oeq float [[ABS]], +inf
 ; CHECK-NEXT:    br i1 [[ISINF]], label [[RETURN:%.*]], label [[IF_END:%.*]]
 ; CHECK:       if.end:
-; CHECK-NEXT:    [[CALL:%.*]] = frem nnan float [[F]], 2.000000e+00
+; CHECK-NEXT:    [[CALL:%.*]] = frem float [[F]], 2.000000e+00
 ; CHECK-NEXT:    ret float [[CALL]]
 ; CHECK:       return:
 ; CHECK-NEXT:    ret float 0.000000e+00
@@ -34,7 +34,7 @@ define float @test_const_zero(float %f) {
 ; CHECK-NEXT:    [[ISZERO:%.*]] = fcmp oeq float [[F]], 0.000000e+00
 ; CHECK-NEXT:    br i1 [[ISZERO]], label [[RETURN:%.*]], label [[IF_END:%.*]]
 ; CHECK:       if.end:
-; CHECK-NEXT:    [[CALL:%.*]] = frem nnan float 2.000000e+00, [[F]]
+; CHECK-NEXT:    [[CALL:%.*]] = frem float 2.000000e+00, [[F]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ; CHECK:       return:
 ; CHECK-NEXT:    ret float 0.000000e+00
@@ -67,7 +67,7 @@ define float @test_noinf_nozero(float nofpclass(inf) %f, float nofpclass(zero) %
 ; CHECK-LABEL: define float @test_noinf_nozero(
 ; CHECK-SAME: float nofpclass(inf) [[F:%.*]], float nofpclass(zero) [[G:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CALL:%.*]] = frem nnan float [[F]], [[G]]
+; CHECK-NEXT:    [[CALL:%.*]] = frem float [[F]], [[G]]
 ; CHECK-NEXT:    ret float [[CALL]]
 ;
 entry:
@@ -79,7 +79,7 @@ define double @test_double(double nofpclass(inf) %f, double nofpclass(zero) %g)
 ; CHECK-LABEL: define double @test_double(
 ; CHECK-SAME: double nofpclass(inf) [[F:%.*]], double nofpclass(zero) [[G:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CALL:%.*]] = frem nnan double [[F]], [[G]]
+; CHECK-NEXT:    [[CALL:%.*]] = frem double [[F]], [[G]]
 ; CHECK-NEXT:    ret double [[CALL]]
 ;
 entry:
@@ -91,7 +91,7 @@ define fp128 @test_fp128(fp128 nofpclass(inf) %f, fp128 nofpclass(zero) %g) {
 ; CHECK-LABEL: define fp128 @test_fp128(
 ; CHECK-SAME: fp128 nofpclass(inf) [[F:%.*]], fp128 nofpclass(zero) [[G:%.*]]) {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CALL:%.*]] = frem nnan fp128 [[F]], [[G]]
+; CHECK-NEXT:    [[CALL:%.*]] = frem fp128 [[F]], [[G]]
 ; CHECK-NEXT:    ret fp128 [[CALL]]
 ;
 entry:
@@ -123,6 +123,20 @@ entry:
   ret float %call
 }
 
+define float @test_nan_input_noerrno(float nofpclass(zero) %g, i1 %c) {
+; CHECK-LABEL: define float @test_nan_input_noerrno(
+; CHECK-SAME: float nofpclass(zero) [[G:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[F:%.*]] = select i1 [[C]], float 0x7FF8000000000000, float 1.000000e+00
+; CHECK-NEXT:    [[CALL:%.*]] = frem float [[F]], [[G]]
+; CHECK-NEXT:    ret float [[CALL]]
+;
+entry:
+  %f = select i1 %c, float 0x7FF8000000000000, float 1.0
+  %call = tail call float @fmodf(float %f, float %g)
+  ret float %call
+}
+
 define float @test_nnan(float %f, float %g) {
 ; CHECK-LABEL: define float @test_nnan(
 ; CHECK-SAME: float [[F:%.*]], float [[G:%.*]]) {