[flang-commits] [flang] [flang][runtime] Correct EXw.0 output editing (PR #75121)

[Peter Klausler via flang-commits]

https://github.com/klausler updated https://github.com/llvm/llvm-project/pull/75121

>From 2cce66bf96416da5352715d7074bbe9c86922a4f Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler at nvidia.com>
Date: Mon, 11 Dec 2023 16:28:06 -0800
Subject: [PATCH] [flang][runtime] Correct EXw.0 output editing

A zero 'd' digit count in EX output editing has a meaning that's
distinct from other numeric output editing descriptors, and I
missed this in my initial implementation of the feature.  d==0
means that the runtime should emit hexadecimal digits after the
(hexa)decimal point until all of the rest of them would be zero.
---
 .../flang/Decimal/binary-floating-point.h     |  2 +-
 flang/runtime/edit-output.cpp                 |  5 +-
 .../unittests/Runtime/NumericalFormatTest.cpp | 52 ++++++++++++-------
 3 files changed, 38 insertions(+), 21 deletions(-)

diff --git a/flang/include/flang/Decimal/binary-floating-point.h b/flang/include/flang/Decimal/binary-floating-point.h
index b9346a8585e2d2..d1992819f85aa6 100644
--- a/flang/include/flang/Decimal/binary-floating-point.h
+++ b/flang/include/flang/Decimal/binary-floating-point.h
@@ -143,7 +143,7 @@ class BinaryFloatingPointNumber : public common::RealDetails<BINARY_PRECISION> {
     if (IsNaN() || IsInfinite() || keepBits >= binaryPrecision) {
       return true;
     }
-    int lostBits{binaryPrecision - keepBits};
+    int lostBits{keepBits < binaryPrecision ? binaryPrecision - keepBits : 0};
     RawType lostMask{static_cast<RawType>((RawType{1} << lostBits) - 1)};
     if (RawType lost{static_cast<RawType>(raw_ & lostMask)}; lost != 0) {
       bool increase{false};
diff --git a/flang/runtime/edit-output.cpp b/flang/runtime/edit-output.cpp
index a4ce0b12f91111..32b13a8007d0cd 100644
--- a/flang/runtime/edit-output.cpp
+++ b/flang/runtime/edit-output.cpp
@@ -649,7 +649,7 @@ auto RealOutputEditing<KIND>::ConvertToHexadecimal(
     // x_.binaryPrecision is constant, so / can be used for readability.
     int shift{x_.binaryPrecision - 4};
     typename BinaryFloatingPoint::RawType one{1};
-    auto remaining{(one << shift) - one};
+    auto remaining{(one << x_.binaryPrecision) - one};
     for (int digits{0}; digits < significantDigits; ++digits) {
       if ((flags & decimal::Minimize) && !(fraction & remaining)) {
         break;
@@ -682,7 +682,8 @@ bool RealOutputEditing<KIND>::EditEXOutput(const DataEdit &edit) {
     flags |= decimal::AlwaysSign;
   }
   int editWidth{edit.width.value_or(0)}; // 'w' field
-  if (editWidth == 0 && !edit.digits) { // EX0 (no .d)
+  if ((editWidth == 0 && !edit.digits) || editDigits == 0) {
+    // EX0 or EXw.0
     flags |= decimal::Minimize;
     significantDigits = 28; // enough for 128-bit F.P.
   }
diff --git a/flang/unittests/Runtime/NumericalFormatTest.cpp b/flang/unittests/Runtime/NumericalFormatTest.cpp
index b5b8eb05943732..69637d8c6cb4cf 100644
--- a/flang/unittests/Runtime/NumericalFormatTest.cpp
+++ b/flang/unittests/Runtime/NumericalFormatTest.cpp
@@ -654,28 +654,44 @@ TEST(IOApiTests, FormatDoubleValues) {
               {"(EX24.13,';')", " 0XF.FFFFFFFFFFFF8P+1020;"},
           }},
       {// EX rounding
-          0x3ff1000000000000uLL, // 1.0625
+          0x3ff0100000000000uLL,
           {
-              {"(F7.4,';')", " 1.0625;"},
-              {"(EX9.1,';')", " 0X8.8P-3;"},
-              {"(EX9.0,';')", "  0X8.P-3;"},
-              {"(RN,EX9.0,';')", "  0X8.P-3;"},
-              {"(RU,EX9.0,';')", "  0X9.P-3;"},
-              {"(RD,EX9.0,';')", "  0X8.P-3;"},
-              {"(RZ,EX9.0,';')", "  0X8.P-3;"},
-              {"(RC,EX9.0,';')", "  0X9.P-3;"},
+              {"(F11.8,';')", " 1.00390625;"},
+              {"(EX10.2,';')", " 0X8.08P-3;"},
+              {"(EX10.1,';')", "  0X8.0P-3;"},
+              {"(EX10.0,';')", " 0X8.08P-3;"},
+              {"(EX0.0,';')", "0X8.08P-3;"},
+              {"(EX0,';')", "0X8.08P-3;"},
+              {"(RN,EX10.1,';')", "  0X8.0P-3;"},
+              {"(RU,EX10.1,';')", "  0X8.1P-3;"},
+              {"(RD,EX10.1,';')", "  0X8.0P-3;"},
+              {"(RZ,EX10.1,';')", "  0X8.0P-3;"},
+              {"(RC,EX10.1,';')", "  0X8.1P-3;"},
+              {"(RN,EX10.0,';')", " 0X8.08P-3;"},
+              {"(RU,EX10.0,';')", " 0X8.08P-3;"},
+              {"(RD,EX10.0,';')", " 0X8.08P-3;"},
+              {"(RZ,EX10.0,';')", " 0X8.08P-3;"},
+              {"(RC,EX10.0,';')", " 0X8.08P-3;"},
           }},
       {// EX rounding
-          0xbff1000000000000uLL, // -1.0625
+          0xbff0100000000000uLL,
           {
-              {"(F7.4,';')", "-1.0625;"},
-              {"(EX9.1,';')", "-0X8.8P-3;"},
-              {"(EX9.0,';')", " -0X8.P-3;"},
-              {"(RN,EX9.0,';')", " -0X8.P-3;"},
-              {"(RU,EX9.0,';')", " -0X8.P-3;"},
-              {"(RD,EX9.0,';')", " -0X9.P-3;"},
-              {"(RZ,EX9.0,';')", " -0X8.P-3;"},
-              {"(RC,EX9.0,';')", " -0X9.P-3;"},
+              {"(F11.8,';')", "-1.00390625;"},
+              {"(EX10.2,';')", "-0X8.08P-3;"},
+              {"(EX10.1,';')", " -0X8.0P-3;"},
+              {"(EX10.0,';')", "-0X8.08P-3;"},
+              {"(EX0.0,';')", "-0X8.08P-3;"},
+              {"(EX0,';')", "-0X8.08P-3;"},
+              {"(RN,EX10.1,';')", " -0X8.0P-3;"},
+              {"(RU,EX10.1,';')", " -0X8.0P-3;"},
+              {"(RD,EX10.1,';')", " -0X8.1P-3;"},
+              {"(RZ,EX10.1,';')", " -0X8.0P-3;"},
+              {"(RC,EX10.1,';')", " -0X8.1P-3;"},
+              {"(RN,EX10.0,';')", "-0X8.08P-3;"},
+              {"(RU,EX10.0,';')", "-0X8.08P-3;"},
+              {"(RD,EX10.0,';')", "-0X8.08P-3;"},
+              {"(RZ,EX10.0,';')", "-0X8.08P-3;"},
+              {"(RC,EX10.0,';')", "-0X8.08P-3;"},
           }},
   };