[flang-commits] [flang] [flang][runtime] Add limit check to MOD/MODULO (PR #80026)

[Peter Klausler via flang-commits]

https://github.com/klausler updated https://github.com/llvm/llvm-project/pull/80026

>From 4741e458c7440c9f7060e258825b0f76a89a4ba0 Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler at nvidia.com>
Date: Tue, 30 Jan 2024 08:11:31 -0800
Subject: [PATCH] [flang][runtime] Add limit check to MOD/MODULO

When testing the arguments to see whether they are integers,
check first that they are within the maximum range of a
64-bit integer; otherwise, a value of larger magnitude
will set an invalid operand exception flag.
---
 flang/runtime/numeric.cpp | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/flang/runtime/numeric.cpp b/flang/runtime/numeric.cpp
index 8e512aff1ea1d..ede00d69f20e2 100644
--- a/flang/runtime/numeric.cpp
+++ b/flang/runtime/numeric.cpp
@@ -145,14 +145,19 @@ inline RT_API_ATTRS T RealMod(
   } else if (std::isinf(p)) {
     return a;
   }
-  if (auto aInt{static_cast<std::int64_t>(a)}; a == aInt) {
-    if (auto pInt{static_cast<std::int64_t>(p)}; p == pInt) {
-      // Fast exact case for integer operands
-      auto mod{aInt - (aInt / pInt) * pInt};
-      if (IS_MODULO && (aInt > 0) != (pInt > 0)) {
-        mod += pInt;
+  T aAbs{std::abs(a)};
+  T pAbs{std::abs(p)};
+  if (aAbs <= static_cast<T>(std::numeric_limits<std::int64_t>::max()) &&
+      pAbs <= static_cast<T>(std::numeric_limits<std::int64_t>::max())) {
+    if (auto aInt{static_cast<std::int64_t>(a)}; a == aInt) {
+      if (auto pInt{static_cast<std::int64_t>(p)}; p == pInt) {
+        // Fast exact case for integer operands
+        auto mod{aInt - (aInt / pInt) * pInt};
+        if (IS_MODULO && (aInt > 0) != (pInt > 0)) {
+          mod += pInt;
+        }
+        return static_cast<T>(mod);
       }
-      return static_cast<T>(mod);
     }
   }
   if constexpr (std::is_same_v<T, float> || std::is_same_v<T, double> ||
@@ -183,9 +188,8 @@ inline RT_API_ATTRS T RealMod(
     // what's left is the desired remainder.  This is basically
     // the same algorithm as arbitrary precision binary long division,
     // discarding the quotient.
-    T tmp{std::abs(a)};
-    T pAbs{std::abs(p)};
-    for (T adj{SetExponent(pAbs, Exponent<int>(tmp))}; tmp >= pAbs; adj /= 2) {
+    T tmp{aAbs};
+    for (T adj{SetExponent(pAbs, Exponent<int>(aAbs))}; tmp >= pAbs; adj /= 2) {
       if (tmp >= adj) {
         tmp -= adj;
         if (tmp == 0) {