[llvm] [libc] [libc][WIP] Move printf long double to simple calc (PR #75414)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Dec 20 13:00:42 PST 2023
https://github.com/michaelrj-google updated https://github.com/llvm/llvm-project/pull/75414
>From 0796ece3abeccac7b94eed99c13676355b9aab2d Mon Sep 17 00:00:00 2001
From: Michael Jones <michaelrj at google.com>
Date: Wed, 13 Dec 2023 17:03:50 -0800
Subject: [PATCH 1/3] [libc] Move printf long double to simple calc
The Ryu algorithm is very fast with its table, but that table grows too
large for long doubles. This patch adds a method of calculating the
digits of long doubles using just wide integers and fast modulo
operations. This results in significant performance improvements vs the
previous int calc mode, while taking up a similar amound of peak memory.
It will be slow in some %e/%g cases, but reasonable fast for %f with no
loss of accuracy.
---
libc/src/__support/UInt.h | 2 +
libc/src/__support/float_to_string.h | 230 ++++++++++++++++++++++++---
libc/test/src/stdio/sprintf_test.cpp | 31 ++++
3 files changed, 237 insertions(+), 26 deletions(-)
diff --git a/libc/src/__support/UInt.h b/libc/src/__support/UInt.h
index cfd495c5861851..e90535358d6aac 100644
--- a/libc/src/__support/UInt.h
+++ b/libc/src/__support/UInt.h
@@ -448,6 +448,8 @@ template <size_t Bits, bool Signed> struct BigInt {
// pos is the index of the current 64-bit chunk that we are processing.
size_t pos = WORDCOUNT;
+ // TODO: look into if constexpr(Bits > 256) skip leading zeroes.
+
for (size_t q_pos = WORDCOUNT - lower_pos; q_pos > 0; --q_pos) {
// q_pos is 1 + the index of the current 64-bit chunk of the quotient
// being processed.
diff --git a/libc/src/__support/float_to_string.h b/libc/src/__support/float_to_string.h
index 923633e3d207f5..efee38673db3dd 100644
--- a/libc/src/__support/float_to_string.h
+++ b/libc/src/__support/float_to_string.h
@@ -17,6 +17,7 @@
#include "src/__support/UInt.h"
#include "src/__support/common.h"
#include "src/__support/libc_assert.h"
+#include "src/__support/macros/attributes.h"
// This file has 5 compile-time flags to allow the user to configure the float
// to string behavior. These allow the user to select which 2 of the 3 useful
@@ -64,6 +65,8 @@
// long doubles are rarely used and the normal Ryu Printf table is very fast
// for doubles.
+#undef LIBC_COPT_FLOAT_TO_STR_USE_MEGA_LONG_DOUBLE_TABLE
+
#ifdef LIBC_COPT_FLOAT_TO_STR_USE_MEGA_LONG_DOUBLE_TABLE
#include "src/__support/ryu_long_double_constants.h"
#elif !defined(LIBC_COPT_FLOAT_TO_STR_NO_TABLE)
@@ -607,36 +610,208 @@ class FloatToString {
#ifndef LIBC_LONG_DOUBLE_IS_FLOAT64
// --------------------------- LONG DOUBLE FUNCTIONS ---------------------------
-template <>
-LIBC_INLINE constexpr size_t FloatToString<long double>::get_positive_blocks() {
- if (exponent >= -FRACTION_LEN) {
- const uint32_t idx =
- exponent < 0
- ? 0
- : static_cast<uint32_t>(exponent + (IDX_SIZE - 1)) / IDX_SIZE;
- const uint32_t len = internal::length_for_num(idx * IDX_SIZE, FRACTION_LEN);
- return len;
- } else {
- return 0;
+template <> class FloatToString<long double> {
+ fputil::FPBits<long double> float_bits;
+ bool is_negative = 0;
+ int exponent = 0;
+ FloatProp::StorageType mantissa = 0;
+
+ static constexpr int FRACTION_LEN = fputil::FPBits<long double>::FRACTION_LEN;
+ static constexpr int EXP_BIAS = fputil::FPBits<long double>::EXP_BIAS;
+
+ static constexpr size_t FLOAT_AS_INT_WIDTH = 16384;
+ static constexpr size_t EXTRA_INT_WIDTH = 128;
+
+ cpp::BigInt<FLOAT_AS_INT_WIDTH + EXTRA_INT_WIDTH, false> float_as_int = 0;
+ int int_block_index = 0;
+
+ static constexpr size_t BLOCK_BUFFER_LEN = 560;
+ BlockInt block_buffer[BLOCK_BUFFER_LEN] = {0};
+ size_t block_buffer_valid = 0;
+
+ template <size_t Bits>
+ LIBC_INLINE static constexpr BlockInt
+ grab_digits(cpp::BigInt<Bits, false> &int_num) {
+ BlockInt cur_block = 0;
+ auto wide_result = int_num.div_uint32_times_pow_2(1953125, 9);
+ // the optional only comes into effect when dividing by 0, which will
+ // never happen here. Thus, we just assert that it has value.
+ LIBC_ASSERT(wide_result.has_value());
+ cur_block = static_cast<BlockInt>(wide_result.value());
+ return cur_block;
}
-}
-template <>
-LIBC_INLINE constexpr size_t
-FloatToString<long double>::zero_blocks_after_point() {
+ LIBC_INLINE static constexpr void zero_leading_digits(
+ cpp::BigInt<FLOAT_AS_INT_WIDTH + EXTRA_INT_WIDTH, false> &int_num) {
+ // 64 is the width of the numbers used to internally represent the BigInt
+ for (size_t i = 0; i < EXTRA_INT_WIDTH / 64; ++i) {
+ int_num[i + (FLOAT_AS_INT_WIDTH / 64)] = 0;
+ }
+ }
+
+ LIBC_INLINE constexpr void init_convert() {
+ // This initializes float_as_int, cur_block, and block_buffer.
+
+ float_as_int = mantissa;
+
+ // No calculation necessary for the 0 case.
+ if (mantissa == 0 && exponent == 0) {
+ return;
+ }
+
+ if (exponent > 0) {
+ // if the exponent is positive, then the number is fully above the decimal
+ // point. Shift left by exponent to get the integer representation of this
+ // number.
+ float_as_int.shift_left(exponent);
+ int_block_index = 0;
+
+ while (float_as_int > 0) {
+ BlockInt cur_block = grab_digits(float_as_int);
+ block_buffer[int_block_index] = cur_block;
+ ++int_block_index;
+ }
+ block_buffer_valid = int_block_index;
+
+ } else {
+ // if the exponent not positive, then the number is at least partially
+ // below the decimal point. Shift left to make the int a fixed point
+ // representation with the decimal point after the top EXTRA_INT_WIDTH
+ // bits.
+ const int SHIFT_AMOUNT = FLOAT_AS_INT_WIDTH + exponent;
+ static_assert(EXTRA_INT_WIDTH >= sizeof(long double) * 8);
+ float_as_int.shift_left(SHIFT_AMOUNT);
+
+ // If there are still digits above the decimal point, handle those.
+ if (float_as_int.clz() < EXTRA_INT_WIDTH) {
+ cpp::BigInt<EXTRA_INT_WIDTH, false> above_decimal_point =
+ float_as_int >> FLOAT_AS_INT_WIDTH;
+
+ size_t positive_int_block_index = 0;
+ while (above_decimal_point > 0) {
+ BlockInt cur_block = grab_digits(above_decimal_point);
+ block_buffer[positive_int_block_index] = cur_block;
+ ++positive_int_block_index;
+ }
+ block_buffer_valid = positive_int_block_index;
+
+ // Zero all digits above the decimal point.
+ zero_leading_digits(float_as_int);
+ int_block_index = 0;
+ }
+ }
+ }
+
+public:
+ LIBC_INLINE constexpr FloatToString(long double init_float)
+ : float_bits(init_float) {
+ is_negative = float_bits.get_sign();
+ exponent = float_bits.get_explicit_exponent();
+ mantissa = float_bits.get_explicit_mantissa();
+
+ // Adjust for the width of the mantissa.
+ exponent -= FRACTION_LEN;
+
+ this->init_convert();
+ }
+
+ LIBC_INLINE constexpr size_t get_positive_blocks() {
+ if (exponent >= -FRACTION_LEN) {
+ const uint32_t idx =
+ exponent < 0
+ ? 0
+ : static_cast<uint32_t>(exponent + (IDX_SIZE - 1)) / IDX_SIZE;
+ const uint32_t len = internal::length_for_num(idx * IDX_SIZE, FRACTION_LEN);
+ return len;
+ } else {
+ return 0;
+ }
+ }
+
+ LIBC_INLINE constexpr size_t zero_blocks_after_point() {
#ifdef LIBC_COPT_FLOAT_TO_STR_USE_MEGA_LONG_DOUBLE_TABLE
- return MIN_BLOCK_2[-exponent / IDX_SIZE];
+ return MIN_BLOCK_2[-exponent / IDX_SIZE];
#else
- return 0;
- // TODO (michaelrj): Find a good algorithm for this that doesn't use a table.
+ if (exponent < -FRACTION_LEN) {
+ const int pos_exp = -exponent - 1;
+ const uint32_t pos_idx =
+ static_cast<uint32_t>(pos_exp + (IDX_SIZE - 1)) / IDX_SIZE;
+ const int32_t pos_len = ((internal::ceil_log10_pow2(pos_idx * IDX_SIZE) -
+ internal::ceil_log10_pow2(FRACTION_LEN + 1)) /
+ BLOCK_SIZE) -
+ 1;
+ const uint32_t len = static_cast<uint32_t>(pos_len > 0 ? pos_len : 0);
+ return len;
+ }
+ return 0;
+
#endif
-}
+ }
-template <>
-LIBC_INLINE constexpr bool FloatToString<long double>::is_lowest_block(size_t) {
- return false;
-}
+ LIBC_INLINE constexpr bool is_lowest_block(size_t negative_block_index) {
+ // The decimal representation of 2**(-i) will have exactly i digits after
+ // the decimal point.
+ int num_requested_digits =
+ static_cast<int>((negative_block_index + 1) * BLOCK_SIZE);
+ return num_requested_digits > -exponent;
+ }
+
+ LIBC_INLINE constexpr BlockInt get_positive_block(int block_index) {
+ if (exponent < -FRACTION_LEN) {
+ return 0;
+ }
+ if (block_index > static_cast<int>(block_buffer_valid) || block_index < 0) {
+ return 0;
+ }
+
+ return block_buffer[block_index];
+ }
+
+ LIBC_INLINE constexpr BlockInt get_negative_block(int negative_block_index) {
+ if (exponent >= 0) {
+ return 0;
+ }
+
+ // negative_block_index starts at 0 with the first block after the decimal
+ // point, and 1 with the second and so on. This converts to the same
+ // block_index used everywhere else.
+
+ int block_index = -1 - negative_block_index;
+
+ // If we're currently after the requested block (remember these are
+ // negative indices) the reset the number to the start. This is only
+ // likely to happen in %g calls. This will also reset int_block_index.
+ if (block_index > int_block_index) {
+ init_convert();
+ }
+
+ // LIBC_ASSERT(block_index >= int_block_index);
+
+ // If we are currently before the requested block. Step until we reach the
+ // requested block. This is likely to only be one step.
+ while (block_index < int_block_index) {
+ zero_leading_digits(float_as_int);
+ float_as_int.mul(1000000000);
+ --int_block_index;
+ }
+
+ // We're currently on the requested block, return the current block.
+ BlockInt cur_block =
+ static_cast<BlockInt>(float_as_int >> FLOAT_AS_INT_WIDTH);
+ return cur_block;
+ }
+
+ LIBC_INLINE constexpr BlockInt get_block(int block_index) {
+ if (block_index >= 0) {
+ return get_positive_block(block_index);
+ } else {
+ return get_negative_block(-1 - block_index);
+ }
+ }
+};
+
+/*
template <>
LIBC_INLINE constexpr BlockInt
FloatToString<long double>::get_positive_block(int block_index) {
@@ -729,8 +904,11 @@ FloatToString<long double>::get_negative_block(int block_index) {
// ----------------------------- INT CALC MODE -----------------------------
const int32_t SHIFT_CONST = CALC_SHIFT_CONST;
- const uint64_t TEN_BLOCKS = (block_index + 1) * BLOCK_SIZE;
- const uint64_t MAX_INT_SIZE = internal::log2_pow5(TEN_BLOCKS);
+ const uint64_t NUM_FIVES = (block_index + 1) * BLOCK_SIZE;
+ // Round MAX_INT_SIZE up to the nearest 64 (adding 1 because log2_pow5
+ // implicitly rounds down).
+ const uint64_t MAX_INT_SIZE =
+ ((internal::log2_pow5(NUM_FIVES) / 64) + 1) * 64;
if (MAX_INT_SIZE < 1024) {
val = internal::get_table_negative<1024>(idx * IDX_SIZE, block_index + 1);
@@ -756,7 +934,7 @@ FloatToString<long double>::get_negative_block(int block_index) {
return 0;
}
}
-
+*/
#endif // LIBC_LONG_DOUBLE_IS_FLOAT64
} // namespace LIBC_NAMESPACE
diff --git a/libc/test/src/stdio/sprintf_test.cpp b/libc/test/src/stdio/sprintf_test.cpp
index 344853beaf9fa7..5a18b1fd88d721 100644
--- a/libc/test/src/stdio/sprintf_test.cpp
+++ b/libc/test/src/stdio/sprintf_test.cpp
@@ -1050,6 +1050,37 @@ TEST_F(LlvmLibcSPrintfTest, FloatDecimalConv) {
"99999999999999999996693535322073426194986990198284960792713"
"91541752018669482644324418977840117055488.000000");
+ written = LIBC_NAMESPACE::sprintf(buff, "%Lf", 0xd.96ed1192687859ap-24L);
+ ASSERT_STREQ_LEN(written, buff, "0.000001");
+
+ written = LIBC_NAMESPACE::sprintf(buff, "%Lf", 10000000000000000.25L);
+ ASSERT_STREQ_LEN(written, buff, "10000000000000000.250000");
+
+ written = LIBC_NAMESPACE::sprintf(buff, "%.510Lf", 0x8p-503L);
+ ASSERT_STREQ_LEN(
+ written, buff,
+ "0."
+ "000000000000000000000000000000000000000000000000000000000000000000000000"
+ "000000000000000000000000000000000000000000000000000000000000000000000000"
+ "000000305493636349960468205197939321361769978940274057232666389361390928"
+ "129162652472045770185723510801522825687515269359046715531785342780428396"
+ "973513311420091788963072442053377285222203558881953188370081650866793017"
+ "948791366338993705251636497892270212003524508209121908744820211960149463"
+ "721109340307985507678283651836204093399373959982767701148986816406250000"
+ "000000");
+
+ written = LIBC_NAMESPACE::sprintf(buff, "%.500Lf", -4327677766926336.0L);
+ ASSERT_STREQ_LEN(
+ written, buff,
+ "-4327677766926336."
+ "000000000000000000000000000000000000000000000000000000000000000000000000"
+ "000000000000000000000000000000000000000000000000000000000000000000000000"
+ "000000000000000000000000000000000000000000000000000000000000000000000000"
+ "000000000000000000000000000000000000000000000000000000000000000000000000"
+ "000000000000000000000000000000000000000000000000000000000000000000000000"
+ "000000000000000000000000000000000000000000000000000000000000000000000000"
+ "00000000000000000000000000000000000000000000000000000000000000000000");
+
written = LIBC_NAMESPACE::sprintf(big_buff, "%Lf", 1e1000L);
ASSERT_STREQ_LEN(
written, big_buff,
>From 0894b8839db06f8ca410889d6b3946e466238447 Mon Sep 17 00:00:00 2001
From: Michael Jones <michaelrj at google.com>
Date: Mon, 18 Dec 2023 16:34:07 -0800
Subject: [PATCH 2/3] cleanup, adjust docs, update build systems
---
libc/config/config.json | 2 +-
libc/docs/dev/printf_behavior.rst | 20 +-
libc/src/__support/float_to_string.h | 276 ++++++------------
.../llvm-project-overlay/libc/BUILD.bazel | 2 -
4 files changed, 107 insertions(+), 193 deletions(-)
diff --git a/libc/config/config.json b/libc/config/config.json
index 77d10d75f36467..6a208cc5566116 100644
--- a/libc/config/config.json
+++ b/libc/config/config.json
@@ -13,7 +13,7 @@
"doc": "Disable handling of %n in printf format string."
},
"LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_MEGA_LONG_DOUBLE_TABLE": {
- "value": true,
+ "value": false,
"doc": "Use large table for better printf long double performance."
}
},
diff --git a/libc/docs/dev/printf_behavior.rst b/libc/docs/dev/printf_behavior.rst
index 29b1b17ebaecb0..52252c61b02546 100644
--- a/libc/docs/dev/printf_behavior.rst
+++ b/libc/docs/dev/printf_behavior.rst
@@ -87,14 +87,25 @@ are not recommended to be adjusted except by persons familiar with the Printf
Ryu Algorithm. Additionally they have no effect when float conversions are
disabled.
+LIBC_COPT_FLOAT_TO_STR_NO_SPECIALIZE_LD
+---------------------------------------
+This flag disables the separate long double conversion implementation. It is
+not based on the Ryu algorithm, instead generating the digits by
+multiplying/dividing the written-out number by 10^9 to get blocks. It's
+significantly faster than INT_CALC, only about 10x slower than MEGA_TABLE,
+and is small in binary size. Its downside is that it always calculates all
+of the digits above the decimal point, making it slightly ineffecient for %e
+calls with large exponents. This is the default. If this flag is not set, no
+other flags will change the long double behavior.
+
LIBC_COPT_FLOAT_TO_STR_USE_MEGA_LONG_DOUBLE_TABLE
-------------------------------------------------
When set, the float to string decimal conversion algorithm will use a larger
table to accelerate long double conversions. This larger table is around 5MB of
-size when compiled. This flag is enabled by default in the CMake.
+size when compiled.
-LIBC_COPT_FLOAT_TO_STR_USE_DYADIC_FLOAT(_LD)
---------------------------------------------
+LIBC_COPT_FLOAT_TO_STR_USE_DYADIC_FLOAT
+---------------------------------------
When set, the float to string decimal conversion algorithm will use dyadic
floats instead of a table when performing floating point conversions. This
results in ~50 digits of accuracy in the result, then zeroes for the remaining
@@ -107,8 +118,7 @@ LIBC_COPT_FLOAT_TO_STR_USE_INT_CALC
When set, the float to string decimal conversion algorithm will use wide
integers instead of a table when performing floating point conversions. This
gives the same results as the table, but is very slow at the extreme ends of
-the long double range. If no flags are set this is the default behavior for
-long double conversions.
+the long double range.
LIBC_COPT_FLOAT_TO_STR_NO_TABLE
-------------------------------
diff --git a/libc/src/__support/float_to_string.h b/libc/src/__support/float_to_string.h
index efee38673db3dd..34b9e16c24d0c9 100644
--- a/libc/src/__support/float_to_string.h
+++ b/libc/src/__support/float_to_string.h
@@ -13,6 +13,7 @@
#include "src/__support/CPP/type_traits.h"
#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/FloatProperties.h"
#include "src/__support/FPUtil/dyadic_float.h"
#include "src/__support/UInt.h"
#include "src/__support/common.h"
@@ -20,12 +21,19 @@
#include "src/__support/macros/attributes.h"
// This file has 5 compile-time flags to allow the user to configure the float
-// to string behavior. These allow the user to select which 2 of the 3 useful
-// properties they want. The useful properties are:
-// 1) Speed of Evaluation
-// 2) Small Size of Binary
-// 3) Centered Output Value
-// These are explained below with the flags that are missing each one.
+// to string behavior. These were used to explore tradeoffs during the design
+// phase, and can still be used to gain specific properties. Unless you
+// specifically know what you're doing, you should leave all these flags off.
+
+// LIBC_COPT_FLOAT_TO_STR_NO_SPECIALIZE_LD
+// This flag disables the separate long double conversion implementation. It is
+// not based on the Ryu algorithm, instead generating the digits by
+// multiplying/dividing the written-out number by 10^9 to get blocks. It's
+// significantly faster than INT_CALC, only about 10x slower than MEGA_TABLE,
+// and is small in binary size. Its downside is that it always calculates all
+// of the digits above the decimal point, making it ineffecient for %e calls
+// with large exponents. If this flag is not set, no other flags will change
+// the long double behavior.
// LIBC_COPT_FLOAT_TO_STR_USE_MEGA_LONG_DOUBLE_TABLE
// The Mega Table is ~5 megabytes when compiled. It lists the constants needed
@@ -34,16 +42,13 @@
// exchange for large binary size.
// LIBC_COPT_FLOAT_TO_STR_USE_DYADIC_FLOAT
-// LIBC_COPT_FLOAT_TO_STR_USE_DYADIC_FLOAT_LD
// Dyadic floats are software floating point numbers, and their accuracy can be
// as high as necessary. This option uses 256 bit dyadic floats to calculate
// the table values that Ryu Printf needs. This is reasonably fast and very
// small compared to the Mega Table, but the 256 bit floats only give accurate
// results for the first ~50 digits of the output. In practice this shouldn't
// be a problem since long doubles are only accurate for ~35 digits, but the
-// trailing values all being 0s may cause brittle tests to fail. The _LD
-// version of this flag only effects the long double calculations, and the
-// other version effects both long double and double.
+// trailing values all being 0s may cause brittle tests to fail.
// LIBC_COPT_FLOAT_TO_STR_USE_INT_CALC
// Integer Calculation uses wide integers to do the calculations for the Ryu
@@ -61,11 +66,8 @@
// Default Config:
// If no flags are set, doubles use the normal (and much more reasonably sized)
-// Ryu Printf table and long doubles use Integer Calculation. This is because
-// long doubles are rarely used and the normal Ryu Printf table is very fast
-// for doubles.
-
-#undef LIBC_COPT_FLOAT_TO_STR_USE_MEGA_LONG_DOUBLE_TABLE
+// Ryu Printf table and long doubles use their specialized implementation. This
+// provides good performance and binary size.
#ifdef LIBC_COPT_FLOAT_TO_STR_USE_MEGA_LONG_DOUBLE_TABLE
#include "src/__support/ryu_long_double_constants.h"
@@ -155,15 +157,17 @@ LIBC_INLINE constexpr uint32_t ceil_log10_pow2(const uint32_t e) {
return log10_pow2(e) + 1;
}
+LIBC_INLINE constexpr uint32_t div_ceil(const uint32_t num,
+ const uint32_t denom) {
+ return (num + (denom - 1)) / denom;
+}
+
// Returns the maximum number of 9 digit blocks a number described by the given
// index (which is ceil(exponent/16)) and mantissa width could need.
LIBC_INLINE constexpr uint32_t length_for_num(const uint32_t idx,
const uint32_t mantissa_width) {
- //+8 to round up when dividing by 9
- return (ceil_log10_pow2(idx) + ceil_log10_pow2(mantissa_width + 1) +
- (BLOCK_SIZE - 1)) /
- BLOCK_SIZE;
- // return (ceil_log10_pow2(16 * idx + mantissa_width) + 8) / 9;
+ return div_ceil(ceil_log10_pow2(idx) + ceil_log10_pow2(mantissa_width + 1),
+ BLOCK_SIZE);
}
// The formula for the table when i is positive (or zero) is as follows:
@@ -430,8 +434,6 @@ class FloatToString {
// Adjust for the width of the mantissa.
exponent -= FRACTION_LEN;
-
- // init_convert();
}
LIBC_INLINE constexpr bool is_nan() { return float_bits.is_nan(); }
@@ -455,6 +457,8 @@ class FloatToString {
// shift_amount = -(c0 - exponent) = c_0 + 16 * ceil(exponent/16) -
// exponent
+ const uint32_t pos_exp = idx * IDX_SIZE;
+
cpp::UInt<MID_INT_SIZE> val;
#if defined(LIBC_COPT_FLOAT_TO_STR_USE_DYADIC_FLOAT)
@@ -465,24 +469,25 @@ class FloatToString {
// ---------------------------- INT CALC MODE ----------------------------
const int32_t SHIFT_CONST = CALC_SHIFT_CONST;
-
const uint64_t MAX_POW_2_SIZE =
- exponent + CALC_SHIFT_CONST - (BLOCK_SIZE * block_index);
+ pos_exp + CALC_SHIFT_CONST - (BLOCK_SIZE * block_index);
const uint64_t MAX_POW_5_SIZE =
internal::log2_pow5(BLOCK_SIZE * block_index);
const uint64_t MAX_INT_SIZE =
(MAX_POW_2_SIZE > MAX_POW_5_SIZE) ? MAX_POW_2_SIZE : MAX_POW_5_SIZE;
if (MAX_INT_SIZE < 1024) {
- val = internal::get_table_positive<1024>(IDX_SIZE * idx, block_index);
+ val = internal::get_table_positive<1024>(pos_exp, block_index);
} else if (MAX_INT_SIZE < 2048) {
- val = internal::get_table_positive<2048>(IDX_SIZE * idx, block_index);
+ val = internal::get_table_positive<2048>(pos_exp, block_index);
} else if (MAX_INT_SIZE < 4096) {
- val = internal::get_table_positive<4096>(IDX_SIZE * idx, block_index);
+ val = internal::get_table_positive<4096>(pos_exp, block_index);
} else if (MAX_INT_SIZE < 8192) {
- val = internal::get_table_positive<8192>(IDX_SIZE * idx, block_index);
+ val = internal::get_table_positive<8192>(pos_exp, block_index);
+ } else if (MAX_INT_SIZE < 16384) {
+ val = internal::get_table_positive<16384>(pos_exp, block_index);
} else {
- val = internal::get_table_positive<16384>(IDX_SIZE * idx, block_index);
+ val = internal::get_table_positive<16384 + 128>(pos_exp, block_index);
}
#else
// ----------------------------- TABLE MODE ------------------------------
@@ -490,9 +495,9 @@ class FloatToString {
val = POW10_SPLIT[POW10_OFFSET[idx] + block_index];
#endif
- const uint32_t shift_amount =
- SHIFT_CONST + (static_cast<uint32_t>(IDX_SIZE) * idx) - exponent;
- const uint32_t digits =
+ const uint32_t shift_amount = SHIFT_CONST + pos_exp - exponent;
+
+ const BlockInt digits =
internal::mul_shift_mod_1e9(mantissa, val, (int32_t)(shift_amount));
return digits;
} else {
@@ -506,35 +511,35 @@ class FloatToString {
cpp::UInt<MID_INT_SIZE> val;
+ const uint32_t pos_exp = idx * IDX_SIZE;
+
#if defined(LIBC_COPT_FLOAT_TO_STR_USE_DYADIC_FLOAT)
// ----------------------- DYADIC FLOAT CALC MODE ------------------------
const int32_t SHIFT_CONST = CALC_SHIFT_CONST;
- val =
- internal::get_table_negative_df<256>(idx * IDX_SIZE, block_index + 1);
+ val = internal::get_table_negative_df<256>(pos_exp, block_index + 1);
#elif defined(LIBC_COPT_FLOAT_TO_STR_USE_INT_CALC)
// ---------------------------- INT CALC MODE ----------------------------
const int32_t SHIFT_CONST = CALC_SHIFT_CONST;
- const uint64_t TEN_BLOCKS = (block_index + 1) * BLOCK_SIZE;
- const uint64_t MAX_INT_SIZE = internal::log2_pow5(TEN_BLOCKS);
+
+ const uint64_t NUM_FIVES = (block_index + 1) * BLOCK_SIZE;
+ // Round MAX_INT_SIZE up to the nearest 64 (adding 1 because log2_pow5
+ // implicitly rounds down).
+ const uint64_t MAX_INT_SIZE =
+ ((internal::log2_pow5(NUM_FIVES) / 64) + 1) * 64;
if (MAX_INT_SIZE < 1024) {
- val =
- internal::get_table_negative<1024>(idx * IDX_SIZE, block_index + 1);
+ val = internal::get_table_negative<1024>(pos_exp, block_index + 1);
} else if (MAX_INT_SIZE < 2048) {
- val =
- internal::get_table_negative<2048>(idx * IDX_SIZE, block_index + 1);
+ val = internal::get_table_negative<2048>(pos_exp, block_index + 1);
} else if (MAX_INT_SIZE < 4096) {
- val =
- internal::get_table_negative<4096>(idx * IDX_SIZE, block_index + 1);
+ val = internal::get_table_negative<4096>(pos_exp, block_index + 1);
} else if (MAX_INT_SIZE < 8192) {
- val =
- internal::get_table_negative<8192>(idx * IDX_SIZE, block_index + 1);
+ val = internal::get_table_negative<8192>(pos_exp, block_index + 1);
} else if (MAX_INT_SIZE < 16384) {
- val = internal::get_table_negative<16384>(idx * IDX_SIZE,
- block_index + 1);
+ val = internal::get_table_negative<16384>(pos_exp, block_index + 1);
} else {
- val = internal::get_table_negative<32768>(idx * IDX_SIZE,
- block_index + 1);
+ val = internal::get_table_negative<16384 + 8192>(pos_exp,
+ block_index + 1);
}
#else
// ----------------------------- TABLE MODE ------------------------------
@@ -552,8 +557,8 @@ class FloatToString {
val = POW10_SPLIT_2[p];
#endif
const int32_t shift_amount =
- SHIFT_CONST + (-exponent - (static_cast<int32_t>(IDX_SIZE) * idx));
- uint32_t digits =
+ SHIFT_CONST + (-exponent - static_cast<int32_t>(pos_exp));
+ BlockInt digits =
internal::mul_shift_mod_1e9(mantissa, val, shift_amount);
return digits;
} else {
@@ -585,12 +590,18 @@ class FloatToString {
// This takes the index of a block after the decimal point (a negative block)
// and return if it's sure that all of the digits after it are zero.
- LIBC_INLINE constexpr bool is_lowest_block(size_t block_index) {
+ LIBC_INLINE constexpr bool is_lowest_block(size_t negative_block_index) {
#ifdef LIBC_COPT_FLOAT_TO_STR_NO_TABLE
- return false;
+ // The decimal representation of 2**(-i) will have exactly i digits after
+ // the decimal point.
+ int num_requested_digits =
+ static_cast<int>((negative_block_index + 1) * BLOCK_SIZE);
+
+ return num_requested_digits > -exponent;
#else
const int32_t idx = -exponent / IDX_SIZE;
- const size_t p = POW10_OFFSET_2[idx] + block_index - MIN_BLOCK_2[idx];
+ const size_t p =
+ POW10_OFFSET_2[idx] + negative_block_index - MIN_BLOCK_2[idx];
// If the remaining digits are all 0, then this is the lowest block.
return p >= POW10_OFFSET_2[idx + 1];
#endif
@@ -598,16 +609,26 @@ class FloatToString {
LIBC_INLINE constexpr size_t zero_blocks_after_point() {
#ifdef LIBC_COPT_FLOAT_TO_STR_NO_TABLE
+ if (exponent < -MANT_WIDTH) {
+ const int pos_exp = -exponent - 1;
+ const uint32_t pos_idx =
+ static_cast<uint32_t>(pos_exp + (IDX_SIZE - 1)) / IDX_SIZE;
+ const int32_t pos_len = ((internal::ceil_log10_pow2(pos_idx * IDX_SIZE) -
+ internal::ceil_log10_pow2(MANT_WIDTH + 1)) /
+ BLOCK_SIZE) -
+ 1;
+ const uint32_t len = static_cast<uint32_t>(pos_len > 0 ? pos_len : 0);
+ return len;
+ }
return 0;
- // TODO (michaelrj): Find a good algorithm for this that doesn't use a
- // table.
#else
return MIN_BLOCK_2[-exponent / IDX_SIZE];
#endif
}
};
-#ifndef LIBC_LONG_DOUBLE_IS_FLOAT64
+#if !defined(LIBC_LONG_DOUBLE_IS_FLOAT64) && \
+ !defined(LIBC_COPT_FLOAT_TO_STR_NO_SPECIALIZE_LD)
// --------------------------- LONG DOUBLE FUNCTIONS ---------------------------
template <> class FloatToString<long double> {
@@ -619,13 +640,21 @@ template <> class FloatToString<long double> {
static constexpr int FRACTION_LEN = fputil::FPBits<long double>::FRACTION_LEN;
static constexpr int EXP_BIAS = fputil::FPBits<long double>::EXP_BIAS;
- static constexpr size_t FLOAT_AS_INT_WIDTH = 16384;
- static constexpr size_t EXTRA_INT_WIDTH = 128;
+ // static constexpr size_t FLOAT_AS_INT_WIDTH = 16384;
+ static constexpr size_t FLOAT_AS_INT_WIDTH =
+ internal::div_ceil(fputil::FPBits<long double>::MAX_EXPONENT -
+ fputil::FPBits<long double>::EXPONENT_BIAS,
+ 64) *
+ 64;
+ // static constexpr size_t EXTRA_INT_WIDTH = 128;
+ static constexpr size_t EXTRA_INT_WIDTH =
+ internal::div_ceil(sizeof(long double) * 8, 64) * 64;
cpp::BigInt<FLOAT_AS_INT_WIDTH + EXTRA_INT_WIDTH, false> float_as_int = 0;
int int_block_index = 0;
- static constexpr size_t BLOCK_BUFFER_LEN = 560;
+ static constexpr size_t BLOCK_BUFFER_LEN =
+ internal::div_ceil(internal::log10_pow2(FLOAT_AS_INT_WIDTH), BLOCK_SIZE);
BlockInt block_buffer[BLOCK_BUFFER_LEN] = {0};
size_t block_buffer_valid = 0;
@@ -786,7 +815,7 @@ template <> class FloatToString<long double> {
init_convert();
}
- // LIBC_ASSERT(block_index >= int_block_index);
+ LIBC_ASSERT(block_index <= int_block_index);
// If we are currently before the requested block. Step until we reach the
// requested block. This is likely to only be one step.
@@ -811,131 +840,8 @@ template <> class FloatToString<long double> {
}
};
-/*
-template <>
-LIBC_INLINE constexpr BlockInt
-FloatToString<long double>::get_positive_block(int block_index) {
- if (exponent >= -FRACTION_LEN) {
-
- // idx is ceil(exponent/16) or 0 if exponent is negative. This is used to
- // find the coarse section of the POW10_SPLIT table that will be used to
- // calculate the 9 digit window, as well as some other related values.
- const uint32_t idx =
- exponent < 0
- ? 0
- : static_cast<uint32_t>(exponent + (IDX_SIZE - 1)) / IDX_SIZE;
- const uint32_t pos_exp = idx * IDX_SIZE;
-
- // shift_amount = -(c0 - exponent) = c_0 + 16 * ceil(exponent/16) - exponent
-
- cpp::UInt<MID_INT_SIZE> val;
-#ifdef LIBC_COPT_FLOAT_TO_STR_USE_MEGA_LONG_DOUBLE_TABLE
- // ------------------------------ TABLE MODE -------------------------------
- const int32_t SHIFT_CONST = TABLE_SHIFT_CONST;
- val = POW10_SPLIT[POW10_OFFSET[idx] + block_index];
-
-#elif defined(LIBC_COPT_FLOAT_TO_STR_USE_DYADIC_FLOAT) || \
- defined(LIBC_COPT_FLOAT_TO_STR_USE_DYADIC_FLOAT_LD)
- // ------------------------ DYADIC FLOAT CALC MODE -------------------------
- const int32_t SHIFT_CONST = CALC_SHIFT_CONST;
- val = internal::get_table_positive_df<256>(pos_exp, block_index);
-#else
- // ----------------------------- INT CALC MODE -----------------------------
- const int32_t SHIFT_CONST = CALC_SHIFT_CONST;
- const uint64_t MAX_POW_2_SIZE =
- pos_exp + CALC_SHIFT_CONST - (BLOCK_SIZE * block_index);
- const uint64_t MAX_POW_5_SIZE =
- internal::log2_pow5(BLOCK_SIZE * block_index);
- const uint64_t MAX_INT_SIZE =
- (MAX_POW_2_SIZE > MAX_POW_5_SIZE) ? MAX_POW_2_SIZE : MAX_POW_5_SIZE;
-
- if (MAX_INT_SIZE < 1024) {
- val = internal::get_table_positive<1024>(pos_exp, block_index);
- } else if (MAX_INT_SIZE < 2048) {
- val = internal::get_table_positive<2048>(pos_exp, block_index);
- } else if (MAX_INT_SIZE < 4096) {
- val = internal::get_table_positive<4096>(pos_exp, block_index);
- } else if (MAX_INT_SIZE < 8192) {
- val = internal::get_table_positive<8192>(pos_exp, block_index);
- } else if (MAX_INT_SIZE < 16384) {
- val = internal::get_table_positive<16384>(pos_exp, block_index);
- } else {
- val = internal::get_table_positive<16384 + 128>(pos_exp, block_index);
- }
-#endif
- const uint32_t shift_amount = SHIFT_CONST + pos_exp - exponent;
-
- const BlockInt digits =
- internal::mul_shift_mod_1e9(mantissa, val, (int32_t)(shift_amount));
- return digits;
- } else {
- return 0;
- }
-}
-
-template <>
-LIBC_INLINE constexpr BlockInt
-FloatToString<long double>::get_negative_block(int block_index) {
- if (exponent < 0) {
- const int32_t idx = -exponent / IDX_SIZE;
-
- cpp::UInt<MID_INT_SIZE> val;
-#ifdef LIBC_COPT_FLOAT_TO_STR_USE_MEGA_LONG_DOUBLE_TABLE
- // ------------------------------ TABLE MODE -------------------------------
- const int32_t SHIFT_CONST = TABLE_SHIFT_CONST;
-
- // if the requested block is zero
- if (block_index < MIN_BLOCK_2[idx]) {
- return 0;
- }
- const uint32_t p = POW10_OFFSET_2[idx] + block_index - MIN_BLOCK_2[idx];
- // If every digit after the requested block is zero.
- if (p >= POW10_OFFSET_2[idx + 1]) {
- return 0;
- }
- val = POW10_SPLIT_2[p];
-#elif defined(LIBC_COPT_FLOAT_TO_STR_USE_DYADIC_FLOAT) || \
- defined(LIBC_COPT_FLOAT_TO_STR_USE_DYADIC_FLOAT_LD)
- // ------------------------ DYADIC FLOAT CALC MODE -------------------------
- const int32_t SHIFT_CONST = CALC_SHIFT_CONST;
-
- val = internal::get_table_negative_df<256>(idx * IDX_SIZE, block_index + 1);
-#else // table mode
- // ----------------------------- INT CALC MODE -----------------------------
- const int32_t SHIFT_CONST = CALC_SHIFT_CONST;
-
- const uint64_t NUM_FIVES = (block_index + 1) * BLOCK_SIZE;
- // Round MAX_INT_SIZE up to the nearest 64 (adding 1 because log2_pow5
- // implicitly rounds down).
- const uint64_t MAX_INT_SIZE =
- ((internal::log2_pow5(NUM_FIVES) / 64) + 1) * 64;
-
- if (MAX_INT_SIZE < 1024) {
- val = internal::get_table_negative<1024>(idx * IDX_SIZE, block_index + 1);
- } else if (MAX_INT_SIZE < 2048) {
- val = internal::get_table_negative<2048>(idx * IDX_SIZE, block_index + 1);
- } else if (MAX_INT_SIZE < 4096) {
- val = internal::get_table_negative<4096>(idx * IDX_SIZE, block_index + 1);
- } else if (MAX_INT_SIZE < 8192) {
- val = internal::get_table_negative<8192>(idx * IDX_SIZE, block_index + 1);
- } else if (MAX_INT_SIZE < 16384) {
- val =
- internal::get_table_negative<16384>(idx * IDX_SIZE, block_index + 1);
- } else {
- val = internal::get_table_negative<16384 + 8192>(idx * IDX_SIZE,
- block_index + 1);
- }
-#endif
- const int32_t shift_amount =
- SHIFT_CONST + (-exponent - static_cast<int>(IDX_SIZE * idx));
- BlockInt digits = internal::mul_shift_mod_1e9(mantissa, val, shift_amount);
- return digits;
- } else {
- return 0;
- }
-}
-*/
-#endif // LIBC_LONG_DOUBLE_IS_FLOAT64
+#endif // !LIBC_LONG_DOUBLE_IS_FLOAT64 &&
+ // !LIBC_COPT_FLOAT_TO_STR_NO_SPECIALIZE_LD
} // namespace LIBC_NAMESPACE
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index d4f2c078db79d9..104e63f652162f 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -25,7 +25,6 @@ licenses(["notice"])
PRINTF_COPTS = [
"LIBC_COPT_STDIO_USE_SYSTEM_FILE",
- "LIBC_COPT_PRINTF_DISABLE_INDEX_MODE",
"LIBC_COPT_PRINTF_DISABLE_WRITE_INT",
]
@@ -416,7 +415,6 @@ libc_support_library(
"src/__support/ryu_constants.h",
"src/__support/ryu_long_double_constants.h",
],
- defines = ["LIBC_COPT_FLOAT_TO_STR_USE_MEGA_LONG_DOUBLE_TABLE"],
deps = [
":__support_common",
":__support_cpp_type_traits",
>From 7f4c05c3d415ee13cb94f3608df0fe76dbb21b3e Mon Sep 17 00:00:00 2001
From: Michael Jones <michaelrj at google.com>
Date: Wed, 20 Dec 2023 13:00:20 -0800
Subject: [PATCH 3/3] Finish rebase
---
libc/src/__support/float_to_string.h | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/libc/src/__support/float_to_string.h b/libc/src/__support/float_to_string.h
index 34b9e16c24d0c9..7c04b892266f6c 100644
--- a/libc/src/__support/float_to_string.h
+++ b/libc/src/__support/float_to_string.h
@@ -642,8 +642,8 @@ template <> class FloatToString<long double> {
// static constexpr size_t FLOAT_AS_INT_WIDTH = 16384;
static constexpr size_t FLOAT_AS_INT_WIDTH =
- internal::div_ceil(fputil::FPBits<long double>::MAX_EXPONENT -
- fputil::FPBits<long double>::EXPONENT_BIAS,
+ internal::div_ceil(fputil::FPBits<long double>::MAX_BIASED_EXPONENT -
+ FloatProp::EXP_BIAS,
64) *
64;
// static constexpr size_t EXTRA_INT_WIDTH = 128;
@@ -750,7 +750,8 @@ template <> class FloatToString<long double> {
exponent < 0
? 0
: static_cast<uint32_t>(exponent + (IDX_SIZE - 1)) / IDX_SIZE;
- const uint32_t len = internal::length_for_num(idx * IDX_SIZE, FRACTION_LEN);
+ const uint32_t len =
+ internal::length_for_num(idx * IDX_SIZE, FRACTION_LEN);
return len;
} else {
return 0;
More information about the llvm-commits
mailing list