[llvm-branch-commits] [libc] [llvm] [libc][math] Expand polyeval.h with array, loop, and Estrin evaluation. (PR #243523)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Nov 17 16:11:38 PST 2026
https://github.com/lntue updated https://github.com/llvm/llvm-project/pull/243523
>From b9828af1f01f618d1c5240f454a5d1bb7703fc1e Mon Sep 17 00:00:00 2001
From: Tue Ly <lntue.h at gmail.com>
Date: Wed, 18 Nov 2026 00:09:31 +0000
Subject: [PATCH] [libc][math] Expand polyeval.h with array, loop, and Estrin
evaluation.
Add array-based, non-unrolled loop, and right-heavy Estrin polynomial
evaluation helpers to fputil:
- Add array overloads with optional OFFSET template parameter for
polyeval and altpolyeval, using sub_multiply for altpolyeval and
supporting both value (<= sizeof(double)) and const-ref (> sizeof(double))
coefficient types.
- Add loop_polyeval and loop_altpolyeval with LIBC_LOOP_NOUNROLL for
code-size-sensitive evaluations.
- Add estrin_polyeval (array and variadic overloads) using a precomputed
dyadic power table and a minimum-depth right-heavy binary tree.
- Add unit tests in polyeval_test.cpp covering float, double,
DoubleDouble, DyadicFloat<128>, and Frac128 across rounding modes,
constexpr evaluation, and evaluation-order sensitivity.
---
libc/src/__support/FPUtil/CMakeLists.txt | 4 +
libc/src/__support/FPUtil/polyeval.h | 393 +++++++++++--
libc/test/src/__support/FPUtil/CMakeLists.txt | 14 +
.../src/__support/FPUtil/polyeval_test.cpp | 518 ++++++++++++++++++
.../libc/src/__support/FPUtil/BUILD.bazel | 2 +
.../test/src/__support/FPUtil/BUILD.bazel | 13 +
6 files changed, 902 insertions(+), 42 deletions(-)
create mode 100644 libc/test/src/__support/FPUtil/polyeval_test.cpp
diff --git a/libc/src/__support/FPUtil/CMakeLists.txt b/libc/src/__support/FPUtil/CMakeLists.txt
index 96bc657f7d2cb..da8f4b0d2d2f6 100644
--- a/libc/src/__support/FPUtil/CMakeLists.txt
+++ b/libc/src/__support/FPUtil/CMakeLists.txt
@@ -174,7 +174,11 @@ add_header_library(
polyeval.h
DEPENDS
.multiply_add
+ libc.hdr.types.size_t
libc.src.__support.common
+ libc.src.__support.CPP.type_traits
+ libc.src.__support.macros.config
+ libc.src.__support.macros.optimization
)
add_header_library(
diff --git a/libc/src/__support/FPUtil/polyeval.h b/libc/src/__support/FPUtil/polyeval.h
index bada5be4b89cd..018b5bf0eb319 100644
--- a/libc/src/__support/FPUtil/polyeval.h
+++ b/libc/src/__support/FPUtil/polyeval.h
@@ -7,80 +7,389 @@
//===----------------------------------------------------------------------===//
///
/// \file
-/// Common header for polyeval implementations.
+/// Common header for polynomial evaluation implementations.
///
//===----------------------------------------------------------------------===//
#ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_POLYEVAL_H
#define LLVM_LIBC_SRC___SUPPORT_FPUTIL_POLYEVAL_H
-#include "multiply_add.h"
+#include "hdr/types/size_t.h"
#include "src/__support/CPP/type_traits.h"
+#include "src/__support/FPUtil/multiply_add.h"
#include "src/__support/common.h"
#include "src/__support/macros/config.h"
-
-// Evaluate polynomial using Horner's Scheme:
-// With polyeval(x, a_0, a_1, ..., a_n) = a_n * x^n + ... + a_1 * x + a_0, we
-// evaluated it as: a_0 + x * (a_1 + x * ( ... (a_(n-1) + x * a_n) ... ) ) ).
-// We will use FMA instructions if available.
-// Example: to evaluate x^3 + 2*x^2 + 3*x + 4, call
-// polyeval( x, 4.0, 3.0, 2.0, 1.0 )
+#include "src/__support/macros/optimization.h"
namespace LIBC_NAMESPACE_DECL {
namespace fputil {
-template <typename T>
-LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(void *)), T>
-polyeval(const T &, const T &a0) {
- return a0;
+// Evaluate polynomial using Horner's scheme:
+// polyeval(x, a_0, a_1, ..., a_n) = a_0 + x * (a_1 + ... + x * a_n).
+template <typename T, typename... Ts>
+LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(double)), T>
+polyeval([[maybe_unused]] T x, T a0, Ts... a) {
+ static_assert((cpp::is_same_v<T, Ts> && ...),
+ "All coefficients must have the same type T.");
+ if constexpr (sizeof...(Ts) == 0) {
+ return a0;
+ } else {
+ T r = polyeval(x, a...);
+ return multiply_add(x, r, a0);
+ }
}
-template <typename T>
-LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(void *)), T>
-polyeval(T, T a0) {
- return a0;
+template <typename T, typename... Ts>
+LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(double)), T>
+polyeval([[maybe_unused]] const T &x, const T &a0, const Ts &...a) {
+ static_assert((cpp::is_same_v<T, Ts> && ...),
+ "All coefficients must have the same type T.");
+ if constexpr (sizeof...(Ts) == 0) {
+ return a0;
+ } else {
+ T r = polyeval(x, a...);
+ return multiply_add(x, r, a0);
+ }
}
+template <size_t OFFSET = 0, typename T, size_t N>
+LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(double)), T>
+polyeval([[maybe_unused]] T x, const T (&a)[N]) {
+ static_assert(OFFSET < N, "OFFSET must be less than array size N.");
+ if constexpr (OFFSET + 1 == N) {
+ return a[OFFSET];
+ } else {
+ T r = polyeval<OFFSET + 1>(x, a);
+ return multiply_add(x, r, a[OFFSET]);
+ }
+}
+
+template <size_t OFFSET = 0, typename T, size_t N>
+LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(double)), T>
+polyeval([[maybe_unused]] const T &x, const T (&a)[N]) {
+ static_assert(OFFSET < N, "OFFSET must be less than array size N.");
+ if constexpr (OFFSET + 1 == N) {
+ return a[OFFSET];
+ } else {
+ T r = polyeval<OFFSET + 1>(x, a);
+ return multiply_add(x, r, a[OFFSET]);
+ }
+}
+
+// Evaluate alternating polynomial:
+// altpolyeval(x, a_0, a_1, ..., a_n) = a_0 - x * a_1 + x^2 * a_2 - ...
+// + (-1)^n * x^n * a_n.
template <typename T, typename... Ts>
-LIBC_INLINE static constexpr cpp::enable_if_t<(sizeof(T) > sizeof(void *)), T>
-polyeval(const T &x, const T &a0, const Ts &...a) {
- return multiply_add(x, polyeval(x, a...), a0);
+LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(double)), T>
+altpolyeval([[maybe_unused]] T x, T a0, Ts... a) {
+ static_assert((cpp::is_same_v<T, Ts> && ...),
+ "All coefficients must have the same type T.");
+ if constexpr (sizeof...(Ts) == 0) {
+ return a0;
+ } else {
+ T r = altpolyeval(x, a...);
+ return sub_multiply(a0, x, r);
+ }
}
template <typename T, typename... Ts>
-LIBC_INLINE LIBC_CONSTEXPR cpp::enable_if_t<(sizeof(T) <= sizeof(void *)), T>
-polyeval(T x, T a0, Ts... a) {
- return multiply_add(x, polyeval(x, a...), a0);
+LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(double)), T>
+altpolyeval([[maybe_unused]] const T &x, const T &a0, const Ts &...a) {
+ static_assert((cpp::is_same_v<T, Ts> && ...),
+ "All coefficients must have the same type T.");
+ if constexpr (sizeof...(Ts) == 0) {
+ return a0;
+ } else {
+ T r = altpolyeval(x, a...);
+ return sub_multiply(a0, x, r);
+ }
}
-// Evaluating alternating polynomials using subtraction directly.
-// altpolyeval(x, a_0, a_1, ..., a_n) = a_0 - x * a_1 + x^2 * a_2 - ... +
-// + (-1)^n x_n * a_n.
-template <typename T>
-LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(void *)), T>
-altpolyeval(const T &, const T &a0) {
- return a0;
+template <size_t OFFSET = 0, typename T, size_t N>
+LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(double)), T>
+altpolyeval([[maybe_unused]] T x, const T (&a)[N]) {
+ static_assert(OFFSET < N, "OFFSET must be less than array size N.");
+ if constexpr (OFFSET + 1 == N) {
+ return a[OFFSET];
+ } else {
+ T r = altpolyeval<OFFSET + 1>(x, a);
+ return sub_multiply(a[OFFSET], x, r);
+ }
+}
+
+template <size_t OFFSET = 0, typename T, size_t N>
+LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(double)), T>
+altpolyeval([[maybe_unused]] const T &x, const T (&a)[N]) {
+ static_assert(OFFSET < N, "OFFSET must be less than array size N.");
+ if constexpr (OFFSET + 1 == N) {
+ return a[OFFSET];
+ } else {
+ T r = altpolyeval<OFFSET + 1>(x, a);
+ return sub_multiply(a[OFFSET], x, r);
+ }
+}
+
+// Evaluate polynomial using a non-unrolled Horner loop:
+// loop_polyeval<OFFSET>(x, a) = a[OFFSET] + x * (a[OFFSET + 1] + ...
+// + x * a[N - 1]).
+template <size_t OFFSET = 0, typename T, size_t N>
+LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(double)), T>
+loop_polyeval([[maybe_unused]] T x, const T (&a)[N]) {
+ static_assert(OFFSET < N, "OFFSET must be less than array size N.");
+ T r = a[N - 1];
+ if constexpr (OFFSET + 1 < N) {
+ LIBC_LOOP_NOUNROLL
+ for (size_t i = N - 1; i > OFFSET; --i)
+ r = multiply_add(x, r, a[i - 1]);
+ }
+ return r;
+}
+
+template <size_t OFFSET = 0, typename T, size_t N>
+LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(double)), T>
+loop_polyeval([[maybe_unused]] const T &x, const T (&a)[N]) {
+ static_assert(OFFSET < N, "OFFSET must be less than array size N.");
+ T r = a[N - 1];
+ if constexpr (OFFSET + 1 < N) {
+ LIBC_LOOP_NOUNROLL
+ for (size_t i = N - 1; i > OFFSET; --i)
+ r = multiply_add(x, r, a[i - 1]);
+ }
+ return r;
+}
+
+// Evaluate alternating polynomial using a non-unrolled Horner loop:
+// loop_altpolyeval<OFFSET>(x, a) = a[OFFSET] - x * a[OFFSET + 1] + ...
+// + (-1)^(N - 1 - OFFSET) * x^(N - 1 - OFFSET) * a[N - 1].
+template <size_t OFFSET = 0, typename T, size_t N>
+LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(double)), T>
+loop_altpolyeval([[maybe_unused]] T x, const T (&a)[N]) {
+ static_assert(OFFSET < N, "OFFSET must be less than array size N.");
+ T r = a[N - 1];
+ if constexpr (OFFSET + 1 < N) {
+ LIBC_LOOP_NOUNROLL
+ for (size_t i = N - 1; i > OFFSET; --i)
+ r = sub_multiply(a[i - 1], x, r);
+ }
+ return r;
+}
+
+template <size_t OFFSET = 0, typename T, size_t N>
+LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(double)), T>
+loop_altpolyeval([[maybe_unused]] const T &x, const T (&a)[N]) {
+ static_assert(OFFSET < N, "OFFSET must be less than array size N.");
+ T r = a[N - 1];
+ if constexpr (OFFSET + 1 < N) {
+ LIBC_LOOP_NOUNROLL
+ for (size_t i = N - 1; i > OFFSET; --i)
+ r = sub_multiply(a[i - 1], x, r);
+ }
+ return r;
}
+namespace internal {
+
template <typename T>
-LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(void *)), T>
-altpolyeval(T, T a0) {
- return a0;
+using ValOrConstRef =
+ cpp::conditional_t<(sizeof(T) > sizeof(double)), const T &, T>;
+
+// For len >= 2, let 2^h < len <= 2^(h + 1), so tree depth is h + 1 and each
+// subtree has capacity at most 2^h. Return the smallest k such that
+// 2^k >= len - 2^h, giving a right-heavy split:
+// LOW = 2^k <= 2^h, 1 <= HI = len - 2^k <= 2^h.
+// Examples:
+// len = 2, 3 -> k = 0 (1 + 1, 1 + 2)
+// len = 4 -> k = 1 (2 + 2)
+// len = 5 -> k = 0 (1 + 4)
+// len = 6 -> k = 1 (2 + 4)
+// len = 7, 8 -> k = 2 (4 + 3, 4 + 4)
+// len = 9 -> k = 0 (1 + 8)
+// len = 10 -> k = 1 (2 + 8)
+// len = 11, 12 -> k = 2 (4 + 7, 4 + 8)
+LIBC_INLINE constexpr size_t estrin_split_log2(size_t len) {
+ size_t h = 0;
+ while ((size_t(1) << (h + 1)) < len)
+ ++h;
+ size_t rem = len - (size_t(1) << h);
+ size_t k = 0;
+ while ((size_t(1) << k) < rem)
+ ++k;
+ return k;
+}
+
+// Return the largest dyadic power index K needed across the entire right-heavy
+// Estrin tree of length len (for len >= 2). Powers up to x^(2^K) can form a
+// full high subtree of length 2^(K + 1) plus a low subtree of length 2^K,
+// covering len <= 3 * 2^K:
+// len = 2, 3 -> K = 0 (x)
+// len = 4..6 -> K = 1 (x, x^2)
+// len = 7..12 -> K = 2 (x, x^2, x^4)
+// len = 13..24 -> K = 3 (x, x^2, x^4, x^8)
+LIBC_INLINE constexpr size_t estrin_max_power_log2(size_t len) {
+ size_t k = 0;
+ while ((size_t(3) << k) < len)
+ ++k;
+ return k;
+}
+
+// Precompute dyadic powers x^(2^0), x^(2^1), ..., x^(2^K) once by repeated
+// squaring so that all subtrees share x^2, x^4, ..., x^(2^K). Members are
+// initialized in declaration order (`prev` before `x_pow`), computing:
+// EstrinPows<T, 0>: x_pow = x
+// EstrinPows<T, K>: prev = EstrinPows<T, K - 1>(x),
+// x_pow = prev.x_pow * prev.x_pow = x^(2^K)
+// pows.get<I>() returns x^(2^I) for 0 <= I <= K.
+template <typename T, size_t K> struct EstrinPows {
+ EstrinPows<T, K - 1> prev;
+ T x_pow;
+
+ LIBC_ALWAYS_INLINE constexpr explicit EstrinPows(ValOrConstRef<T> x)
+ : prev(x), x_pow(prev.x_pow * prev.x_pow) {}
+
+ template <size_t I>
+ LIBC_ALWAYS_INLINE constexpr ValOrConstRef<T> get() const {
+ if constexpr (I == K)
+ return x_pow;
+ else
+ return prev.template get<I>();
+ }
+};
+
+template <typename T> struct EstrinPows<T, 0> {
+ T x_pow;
+
+ LIBC_ALWAYS_INLINE constexpr explicit EstrinPows(ValOrConstRef<T> x)
+ : x_pow(x) {}
+
+ template <size_t I>
+ LIBC_ALWAYS_INLINE constexpr ValOrConstRef<T> get() const {
+ static_assert(I == 0, "Invalid Estrin power index.");
+ return x_pow;
+ }
+};
+
+// Recursively evaluate P(x) = sum_{i = 0}^{LEN - 1} a[OFFSET + i] * x^i using
+// a right-heavy Estrin binary tree of minimum depth ceil(log2(LEN)):
+// - LEN == 1: a[OFFSET]
+// - LEN == 2: multiply_add(x, a[OFFSET + 1], a[OFFSET])
+// = a[OFFSET] + x * a[OFFSET + 1]
+// - LEN >= 3: let K = estrin_split_log2(LEN) and LOW = 2^K. Split into:
+// lo = estrin_eval_tree<OFFSET, LOW>(pows, a)
+// = sum_{i = 0}^{LOW - 1} a[OFFSET + i] * x^i
+// hi = estrin_eval_tree<OFFSET + LOW, LEN - LOW>(pows, a)
+// = sum_{i = 0}^{LEN - LOW - 1} a[OFFSET + LOW + i] * x^i
+// and combine via multiply_add(x^(2^K), hi, lo) = lo + x^(2^K) * hi.
+//
+// Placing the larger subtree in `hi` minimizes both the highest required power
+// of x and the number of roundings on a[OFFSET] when |x| << 1, while keeping
+// tree depth ceil(log2(LEN)). With x2 = x^2 and x4 = x^4, each (u + p * v)
+// below is evaluated as multiply_add(p, v, u):
+// LEN = 3: a0 + x * (a1 + x * a2)
+// LEN = 4: (a0 + x * a1) + x2 * (a2 + x * a3)
+// LEN = 5: a0 + x * ((a1 + x * a2) + x2 * (a3 + x * a4))
+// LEN = 6: (a0 + x * a1) + x2 * ((a2 + x * a3) + x2 * (a4 + x * a5))
+// LEN = 7: ((a0 + x * a1) + x2 * (a2 + x * a3))
+// + x4 * (a4 + x * (a5 + x * a6))
+// LEN = 8: ((a0 + x * a1) + x2 * (a2 + x * a3))
+// + x4 * ((a4 + x * a5) + x2 * (a6 + x * a7))
+template <size_t OFFSET, size_t LEN, typename T, size_t N, size_t MAX_K>
+LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(double)), T>
+estrin_eval_tree(const EstrinPows<T, MAX_K> &pows, const T (&a)[N]) {
+ if constexpr (LEN == 1) {
+ return a[OFFSET];
+ } else if constexpr (LEN == 2) {
+ return multiply_add(pows.template get<0>(), a[OFFSET + 1], a[OFFSET]);
+ } else {
+ constexpr size_t K = estrin_split_log2(LEN);
+ constexpr size_t LOW = size_t(1) << K;
+ if constexpr (LOW == 1) {
+ T hi = estrin_eval_tree<OFFSET + 1, LEN - 1>(pows, a);
+ return multiply_add(pows.template get<0>(), hi, a[OFFSET]);
+ } else {
+ T lo = estrin_eval_tree<OFFSET, LOW>(pows, a);
+ T hi = estrin_eval_tree<OFFSET + LOW, LEN - LOW>(pows, a);
+ return multiply_add(pows.template get<K>(), hi, lo);
+ }
+ }
+}
+
+template <size_t OFFSET, size_t LEN, typename T, size_t N, size_t MAX_K>
+LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(double)), T>
+estrin_eval_tree(const EstrinPows<T, MAX_K> &pows, const T (&a)[N]) {
+ if constexpr (LEN == 1) {
+ return a[OFFSET];
+ } else if constexpr (LEN == 2) {
+ return multiply_add(pows.template get<0>(), a[OFFSET + 1], a[OFFSET]);
+ } else {
+ constexpr size_t K = estrin_split_log2(LEN);
+ constexpr size_t LOW = size_t(1) << K;
+ if constexpr (LOW == 1) {
+ T hi = estrin_eval_tree<OFFSET + 1, LEN - 1>(pows, a);
+ return multiply_add(pows.template get<0>(), hi, a[OFFSET]);
+ } else {
+ T lo = estrin_eval_tree<OFFSET, LOW>(pows, a);
+ T hi = estrin_eval_tree<OFFSET + LOW, LEN - LOW>(pows, a);
+ return multiply_add(pows.template get<K>(), hi, lo);
+ }
+ }
+}
+
+} // namespace internal
+
+// Evaluate polynomial using Estrin's scheme:
+// estrin_polyeval(x, a_0, a_1, ..., a_n) = a_0 + a_1 * x + ... + a_n * x^n.
+template <size_t OFFSET = 0, typename T, size_t N>
+LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(double)), T>
+estrin_polyeval([[maybe_unused]] T x, const T (&a)[N]) {
+ static_assert(OFFSET < N, "OFFSET must be less than array size N.");
+ constexpr size_t LEN = N - OFFSET;
+ if constexpr (LEN == 1) {
+ return a[OFFSET];
+ } else {
+ constexpr size_t MAX_K = internal::estrin_max_power_log2(LEN);
+ internal::EstrinPows<T, MAX_K> pows(x);
+ return internal::estrin_eval_tree<OFFSET, LEN>(pows, a);
+ }
+}
+
+template <size_t OFFSET = 0, typename T, size_t N>
+LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(double)), T>
+estrin_polyeval([[maybe_unused]] const T &x, const T (&a)[N]) {
+ static_assert(OFFSET < N, "OFFSET must be less than array size N.");
+ constexpr size_t LEN = N - OFFSET;
+ if constexpr (LEN == 1) {
+ return a[OFFSET];
+ } else {
+ constexpr size_t MAX_K = internal::estrin_max_power_log2(LEN);
+ internal::EstrinPows<T, MAX_K> pows(x);
+ return internal::estrin_eval_tree<OFFSET, LEN>(pows, a);
+ }
}
-// TODO: Make use of FMA instructions when using these for floating points.
template <typename T, typename... Ts>
-LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(void *)), T>
-altpolyeval(const T &x, const T &a0, const Ts &...a) {
- T prod = x * altpolyeval(x, a...);
- return a0 - prod;
+LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(double)), T>
+estrin_polyeval([[maybe_unused]] T x, T a0, Ts... a) {
+ static_assert((cpp::is_same_v<T, Ts> && ...),
+ "All coefficients must have the same type T.");
+ if constexpr (sizeof...(Ts) == 0) {
+ return a0;
+ } else {
+ const T coeffs[] = {a0, a...};
+ return estrin_polyeval(x, coeffs);
+ }
}
template <typename T, typename... Ts>
-LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(void *)), T>
-altpolyeval(T x, T a0, Ts... a) {
- T prod = x * altpolyeval(x, a...);
- return a0 - prod;
+LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(double)), T>
+estrin_polyeval([[maybe_unused]] const T &x, const T &a0, const Ts &...a) {
+ static_assert((cpp::is_same_v<T, Ts> && ...),
+ "All coefficients must have the same type T.");
+ if constexpr (sizeof...(Ts) == 0) {
+ return a0;
+ } else {
+ const T coeffs[] = {a0, a...};
+ return estrin_polyeval(x, coeffs);
+ }
}
} // namespace fputil
diff --git a/libc/test/src/__support/FPUtil/CMakeLists.txt b/libc/test/src/__support/FPUtil/CMakeLists.txt
index 99c09bc116a58..c61b3465f9c57 100644
--- a/libc/test/src/__support/FPUtil/CMakeLists.txt
+++ b/libc/test/src/__support/FPUtil/CMakeLists.txt
@@ -76,6 +76,20 @@ add_fp_unittest(
libc.src.__support.frac128
)
+add_fp_unittest(
+ polyeval_test
+ SUITE
+ libc-fputil-tests
+ SRCS
+ polyeval_test.cpp
+ DEPENDS
+ libc.src.__support.FPUtil.double_double
+ libc.src.__support.FPUtil.dyadic_float
+ libc.src.__support.FPUtil.multiply_add
+ libc.src.__support.FPUtil.polyeval
+ libc.src.__support.frac128
+)
+
add_fp_unittest(
rounding_mode_test
SUITE
diff --git a/libc/test/src/__support/FPUtil/polyeval_test.cpp b/libc/test/src/__support/FPUtil/polyeval_test.cpp
new file mode 100644
index 0000000000000..5b23226b41824
--- /dev/null
+++ b/libc/test/src/__support/FPUtil/polyeval_test.cpp
@@ -0,0 +1,518 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Unittests for polyeval, altpolyeval, loop_polyeval, loop_altpolyeval, and
+/// estrin_polyeval.
+///
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/FPUtil/double_double.h"
+#include "src/__support/FPUtil/dyadic_float.h"
+#include "src/__support/FPUtil/multiply_add.h"
+#include "src/__support/FPUtil/polyeval.h"
+#include "src/__support/frac128.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+using LIBC_NAMESPACE::Frac128;
+using LIBC_NAMESPACE::fputil::altpolyeval;
+using LIBC_NAMESPACE::fputil::DoubleDouble;
+using LIBC_NAMESPACE::fputil::DyadicFloat;
+using LIBC_NAMESPACE::fputil::estrin_polyeval;
+using LIBC_NAMESPACE::fputil::loop_altpolyeval;
+using LIBC_NAMESPACE::fputil::loop_polyeval;
+using LIBC_NAMESPACE::fputil::multiply_add;
+using LIBC_NAMESPACE::fputil::polyeval;
+using LIBC_NAMESPACE::fputil::sub_multiply;
+using LIBC_NAMESPACE::fputil::testing::ForceRoundingMode;
+using LIBC_NAMESPACE::fputil::testing::RoundingMode;
+
+static constexpr RoundingMode ROUNDING_MODES[4] = {
+ RoundingMode::Nearest,
+ RoundingMode::Upward,
+ RoundingMode::Downward,
+ RoundingMode::TowardZero,
+};
+
+template <typename T>
+class LlvmLibcPolyEvalTest : public LIBC_NAMESPACE::testing::Test {
+public:
+ void test_polyeval() {
+ constexpr T C[6] = {
+ T(0x1.921fb54442d18p0), T(-0x1.5bf0a8b145769p1),
+ T(0x1.62e42fefa39efp-1), T(-0x1.71547652b82fep-1),
+ T(0x1.a01a01a01a01ap0), T(-0x1.27e4fb7789f5cp0),
+ };
+ constexpr T XS[5] = {
+ T(0.0),
+ T(0.5),
+ T(0x1.3579bdf13579bp-1),
+ T(-0x1.2468ace02468bp-2),
+ T(0x1.1111111111111p0),
+ };
+
+ for (RoundingMode mode : ROUNDING_MODES) {
+ ForceRoundingMode r(mode);
+ if (!r.success)
+ continue;
+ for (T raw_x : XS) {
+ volatile T vx = raw_x;
+ T x = vx;
+
+ EXPECT_FP_EQ(C[0], polyeval(x, C[0]));
+ EXPECT_FP_EQ(C[5], polyeval<5>(x, C));
+ EXPECT_FP_EQ(C[5], loop_polyeval<5>(x, C));
+
+ T h1 = multiply_add(x, C[1], C[0]);
+ EXPECT_FP_EQ(h1, polyeval(x, C[0], C[1]));
+
+ T h2_1 = multiply_add(x, C[2], C[1]);
+ T h2 = multiply_add(x, h2_1, C[0]);
+ EXPECT_FP_EQ(h2, polyeval(x, C[0], C[1], C[2]));
+
+ T t5 = C[5];
+ T t4 = multiply_add(x, t5, C[4]);
+ T t3 = multiply_add(x, t4, C[3]);
+ T t2 = multiply_add(x, t3, C[2]);
+ T t1 = multiply_add(x, t2, C[1]);
+ T t0 = multiply_add(x, t1, C[0]);
+
+ EXPECT_FP_EQ(t0, polyeval(x, C[0], C[1], C[2], C[3], C[4], C[5]));
+ EXPECT_FP_EQ(t0, polyeval(x, C));
+ EXPECT_FP_EQ(t1, polyeval<1>(x, C));
+ EXPECT_FP_EQ(t2, polyeval<2>(x, C));
+ EXPECT_FP_EQ(t4, polyeval<4>(x, C));
+ EXPECT_FP_EQ(t0, loop_polyeval(x, C));
+ EXPECT_FP_EQ(t1, loop_polyeval<1>(x, C));
+ EXPECT_FP_EQ(t2, loop_polyeval<2>(x, C));
+ EXPECT_FP_EQ(t4, loop_polyeval<4>(x, C));
+ }
+ }
+ }
+
+ void test_altpolyeval() {
+ constexpr T EXACT_C[5] = {T(2.0), T(1.5), T(0.75), T(0.25), T(1.125)};
+
+ // Exact dyadic check at x = 0.5:
+ // 2.0 - 0.5 * 1.5 + 0.25 * 0.75 - 0.125 * 0.25 + 0.0625 * 1.125
+ // = (512 - 192 + 48 - 8 + 18) / 256 = 378 / 256 = 189 / 128 = 1.4765625.
+ EXPECT_FP_EQ(T(1.4765625), altpolyeval(T(0.5), EXACT_C));
+ EXPECT_FP_EQ(T(1.4765625), loop_altpolyeval(T(0.5), EXACT_C));
+ EXPECT_FP_EQ(T(1.4765625), altpolyeval(T(0.5), EXACT_C[0], EXACT_C[1],
+ EXACT_C[2], EXACT_C[3], EXACT_C[4]));
+
+ constexpr T C[5] = {
+ T(0x1.921fb54442d18p0), T(0x1.5bf0a8b145769p0),
+ T(0x1.62e42fefa39efp-1), T(0x1.71547652b82fep-2),
+ T(0x1.a01a01a01a01ap-1),
+ };
+ constexpr T XS[5] = {
+ T(0.0),
+ T(0.5),
+ T(0x1.3579bdf13579bp-1),
+ T(-0x1.2468ace02468bp-2),
+ T(0x1.7f3d5b97f3d5cp-1),
+ };
+
+ // Check constexpr coefficient array in round-to-nearest.
+ {
+ ForceRoundingMode r(RoundingMode::Nearest);
+ if (r.success) {
+ volatile T vx = XS[2];
+ T x = vx;
+ T t4 = C[4];
+ T t3 = sub_multiply(C[3], x, t4);
+ T t2 = sub_multiply(C[2], x, t3);
+ T t1 = sub_multiply(C[1], x, t2);
+ T t0 = sub_multiply(C[0], x, t1);
+ EXPECT_FP_EQ(t0, altpolyeval(x, C));
+ EXPECT_FP_EQ(t0, loop_altpolyeval(x, C));
+ }
+ }
+
+ // Load coefficients at runtime when testing directed rounding modes so
+ // compilers without -frounding-math do not fold -C[4] differently between
+ // unrolled altpolyeval and non-unrolled loop_altpolyeval.
+ volatile T vc[5] = {C[0], C[1], C[2], C[3], C[4]};
+ const T rc[5] = {vc[0], vc[1], vc[2], vc[3], vc[4]};
+
+ for (RoundingMode mode : ROUNDING_MODES) {
+ ForceRoundingMode r(mode);
+ if (!r.success)
+ continue;
+ for (T raw_x : XS) {
+ volatile T vx = raw_x;
+ T x = vx;
+
+ EXPECT_FP_EQ(rc[0], altpolyeval(x, rc[0]));
+ EXPECT_FP_EQ(rc[4], altpolyeval<4>(x, rc));
+ EXPECT_FP_EQ(rc[4], loop_altpolyeval<4>(x, rc));
+
+ T t4 = rc[4];
+ T t3 = sub_multiply(rc[3], x, t4);
+ T t2 = sub_multiply(rc[2], x, t3);
+ T t1 = sub_multiply(rc[1], x, t2);
+ T t0 = sub_multiply(rc[0], x, t1);
+
+ EXPECT_FP_EQ(t0, altpolyeval(x, rc[0], rc[1], rc[2], rc[3], rc[4]));
+ EXPECT_FP_EQ(t0, altpolyeval(x, rc));
+ EXPECT_FP_EQ(t1, altpolyeval<1>(x, rc));
+ EXPECT_FP_EQ(t2, altpolyeval<2>(x, rc));
+ EXPECT_FP_EQ(t3, altpolyeval<3>(x, rc));
+ EXPECT_FP_EQ(t0, loop_altpolyeval(x, rc));
+ EXPECT_FP_EQ(t1, loop_altpolyeval<1>(x, rc));
+ EXPECT_FP_EQ(t2, loop_altpolyeval<2>(x, rc));
+ EXPECT_FP_EQ(t3, loop_altpolyeval<3>(x, rc));
+ }
+ }
+ }
+
+ void test_estrin_polyeval() {
+ constexpr T C[13] = {
+ T(0x1.921fb54442d18p0), T(-0x1.5bf0a8b145769p-1),
+ T(0x1.62e42fefa39efp-1), T(-0x1.71547652b82fep-2),
+ T(0x1.a01a01a01a01ap-2), T(-0x1.27e4fb7789f5cp-2),
+ T(0x1.1eed8eff8d898p-1), T(-0x1.b803473f7ad0fp-2),
+ T(0x1.3b2ab6fba4e77p-1), T(-0x1.d8e64b8d4ddadp-2),
+ T(0x1.0c152382d7365p-1), T(-0x1.8f8b83c69a60bp-2),
+ T(0x1.7c5db1052981bp-2),
+ };
+ constexpr T XS[5] = {
+ T(0.0),
+ T(0.5),
+ T(0x1.3579bdf13579bp-1),
+ T(-0x1.2468ace02468bp-2),
+ T(0x1.7f3d5b97f3d5cp-1),
+ };
+
+ for (RoundingMode mode : ROUNDING_MODES) {
+ ForceRoundingMode r(mode);
+ if (!r.success)
+ continue;
+ for (T raw_x : XS) {
+ volatile T vx = raw_x;
+ T x = vx;
+
+ T x2 = x * x;
+ T x4 = x2 * x2;
+ T x8 = x4 * x4;
+
+ // Length 1 (degree 0).
+ EXPECT_FP_EQ(C[0], estrin_polyeval(x, C[0]));
+ EXPECT_FP_EQ(C[12], estrin_polyeval<12>(x, C));
+
+ // Length 2 (degree 1): a0 + x * a1.
+ T p01 = multiply_add(x, C[1], C[0]);
+ EXPECT_FP_EQ(p01, estrin_polyeval(x, C[0], C[1]));
+
+ // Length 3 (degree 2, 1 + 2): a0 + x * (a1 + x * a2).
+ T p12 = multiply_add(x, C[2], C[1]);
+ T e3 = multiply_add(x, p12, C[0]);
+ EXPECT_FP_EQ(e3, estrin_polyeval(x, C[0], C[1], C[2]));
+
+ // Length 4 (degree 3, 2 + 2): (a0 + x * a1) + x2 * (a2 + x * a3).
+ T p23 = multiply_add(x, C[3], C[2]);
+ T e4 = multiply_add(x2, p23, p01);
+ EXPECT_FP_EQ(e4, estrin_polyeval(x, C[0], C[1], C[2], C[3]));
+
+ // Length 5 (degree 4, 1 + 4): a0 + x * ((a1 + x*a2) + x2 * (a3 +
+ // x*a4)).
+ T p34 = multiply_add(x, C[4], C[3]);
+ T e1_4 = multiply_add(x2, p34, p12);
+ T e5 = multiply_add(x, e1_4, C[0]);
+ EXPECT_FP_EQ(e5, estrin_polyeval(x, C[0], C[1], C[2], C[3], C[4]));
+
+ // Length 6 (degree 5, 2 + 4): (a0 + x*a1) + x2 * E4(a2..a5).
+ T p45 = multiply_add(x, C[5], C[4]);
+ T e2_5 = multiply_add(x2, p45, p23);
+ T e6 = multiply_add(x2, e2_5, p01);
+ EXPECT_FP_EQ(e6,
+ estrin_polyeval(x, C[0], C[1], C[2], C[3], C[4], C[5]));
+
+ // Length 7 (degree 6, 4 + 3): E4(a0..a3) + x4 * (a4 + x*(a5 + x*a6)).
+ T p56 = multiply_add(x, C[6], C[5]);
+ T e4_6 = multiply_add(x, p56, C[4]);
+ T e7 = multiply_add(x4, e4_6, e4);
+ EXPECT_FP_EQ(
+ e7, estrin_polyeval(x, C[0], C[1], C[2], C[3], C[4], C[5], C[6]));
+
+ // Length 8 (degree 7, 4 + 4): E4(a0..a3) + x4 * E4(a4..a7).
+ T p67 = multiply_add(x, C[7], C[6]);
+ T e4_7 = multiply_add(x2, p67, p45);
+ T e8 = multiply_add(x4, e4_7, e4);
+ EXPECT_FP_EQ(e8, estrin_polyeval(x, C[0], C[1], C[2], C[3], C[4], C[5],
+ C[6], C[7]));
+
+ // Length 9 (degree 8, 1 + 8): a0 + x * E8(a1..a8).
+ T p78 = multiply_add(x, C[8], C[7]);
+ T e5_8 = multiply_add(x2, p78, p56);
+ T e1_8 = multiply_add(x4, e5_8, e1_4);
+ T e9 = multiply_add(x, e1_8, C[0]);
+ EXPECT_FP_EQ(e9, estrin_polyeval(x, C[0], C[1], C[2], C[3], C[4], C[5],
+ C[6], C[7], C[8]));
+
+ // Slice with OFFSET = 7 (6 elements: C[7..12], 2 + 4).
+ T p9_10 = multiply_add(x, C[10], C[9]);
+ T p11_12 = multiply_add(x, C[12], C[11]);
+ T e9_12 = multiply_add(x2, p11_12, p9_10);
+ T s7_12 = multiply_add(x2, e9_12, p78);
+ EXPECT_FP_EQ(s7_12, estrin_polyeval<7>(x, C));
+
+ // Full array C (length 13, 8 + 5): E8(a0..a7) + x8 * (a8 +
+ // x*E4(a9..a12)).
+ T e8_12 = multiply_add(x, e9_12, C[8]);
+ T e13 = multiply_add(x8, e8_12, e8);
+ EXPECT_FP_EQ(e13, estrin_polyeval(x, C));
+ }
+ }
+ }
+
+ // Verify with half-ULP inputs that wrong tree parenthesizations fail.
+ void test_order_sensitivity() {
+ ForceRoundingMode r(RoundingMode::Nearest);
+ if (!r.success)
+ return;
+
+ // U = 2^(-p) = 0.5 * ulp(1.0), so RN(1.0 + U) = 1.0 (tie to even) whereas
+ // RN(1.0 + c * U) = 1.0 + 2 * U for 1 < c < 3.
+ constexpr T U = (sizeof(T) == sizeof(float)) ? T(0x1p-24) : T(0x1p-53);
+ constexpr T ONE_PLUS_ULP = T(1.0) + T(2.0) * U;
+
+ volatile T vx = T(0.5);
+ T x = vx;
+ T x2 = x * x;
+ T x4 = x2 * x2;
+
+ // LEN = 3: exact = 1 + 1.5 * U.
+ // Right-heavy a0 + x*(a1 + x*a2) = RN(1 + 1.5*U) = 1 + 2*U,
+ // whereas left-heavy (a0 + x*a1) + x2*a2 = RN(RN(1 + U) + 0.5*U) = 1.
+ constexpr T C3[3] = {T(1.0), T(2.0) * U, T(2.0) * U};
+ T left3_lo = multiply_add(x, C3[1], C3[0]);
+ T left3 = multiply_add(x2, C3[2], left3_lo);
+ EXPECT_FP_EQ(T(1.0), left3);
+ EXPECT_FP_EQ(ONE_PLUS_ULP, polyeval(x, C3));
+ EXPECT_FP_EQ(ONE_PLUS_ULP, loop_polyeval(x, C3));
+ EXPECT_FP_EQ(ONE_PLUS_ULP, estrin_polyeval(x, C3));
+
+ // LEN = 4: exact = 1 + 1.25 * U.
+ // Horner: a2 + x*a3 = U -> a1 + x*U = 2.5*U -> RN(1 + 1.25*U) = 1 + 2*U.
+ // Estrin (2 + 2): RN(1 + U) = 1, p23 = U -> RN(1 + 0.25*U) = 1.
+ constexpr T C4[4] = {T(1.0), T(2.0) * U, T(2.0) * U, T(-2.0) * U};
+ EXPECT_FP_EQ(ONE_PLUS_ULP, polyeval(x, C4));
+ EXPECT_FP_EQ(ONE_PLUS_ULP, loop_polyeval(x, C4));
+ EXPECT_FP_EQ(T(1.0), estrin_polyeval(x, C4));
+
+ // LEN = 5: exact = 1 + 1.5 * U.
+ // Right-heavy (1 + 4): E4(2*U, 0, 0, 8*U) = 3*U -> RN(1 + 1.5*U) = 1+2*U.
+ // Left-heavy (4 + 1): E4(1, 2*U, 0, 0) = 1 -> RN(1 + 0.5*U) = 1.
+ constexpr T C5[5] = {T(1.0), T(2.0) * U, T(0.0), T(0.0), T(8.0) * U};
+ T left5_lo = estrin_polyeval(x, C5[0], C5[1], C5[2], C5[3]);
+ T left5 = multiply_add(x4, C5[4], left5_lo);
+ EXPECT_FP_EQ(T(1.0), left5);
+ EXPECT_FP_EQ(ONE_PLUS_ULP, estrin_polyeval(x, C5));
+
+ // LEN = 6: exact = 1 + 1.5 * U.
+ // Right-heavy (2 + 4): p01 = 1, E4(4*U, 0, 8*U, 0) = 6*U
+ // -> RN(1 + 0.25 * 6*U) = 1 + 2*U.
+ // Left-heavy (4 + 2): E4(1, 0, 4*U, 0) = RN(1 + U) = 1
+ // -> RN(1 + 0.0625 * 8*U) = 1.
+ constexpr T C6[6] = {T(1.0), T(0.0), T(4.0) * U,
+ T(0.0), T(8.0) * U, T(0.0)};
+ T left6_lo = estrin_polyeval(x, C6[0], C6[1], C6[2], C6[3]);
+ T left6_hi = multiply_add(x, C6[5], C6[4]);
+ T left6 = multiply_add(x4, left6_hi, left6_lo);
+ EXPECT_FP_EQ(T(1.0), left6);
+ EXPECT_FP_EQ(ONE_PLUS_ULP, estrin_polyeval(x, C6));
+
+ // LEN = 7: 4 + 3 with right-heavy hi = a4 + x*(a5 + x*a6) = 1 + 2*U,
+ // whereas left-heavy hi = (a4 + x*a5) + x2*a6 = 1.
+ constexpr T C7[7] = {T(0.0), T(0.0), T(0.0), T(0.0),
+ T(1.0), T(2.0) * U, T(2.0) * U};
+ EXPECT_FP_EQ(T(0.0625) * ONE_PLUS_ULP, estrin_polyeval(x, C7));
+ }
+
+ void test_constexpr() {
+#if LIBC_HAS_BUILTIN_IS_CONSTANT_EVALUATED
+ constexpr T C[4] = {T(2.0), T(1.5), T(0.75), T(0.25)};
+ // At x = 0.5: 2.0 + 0.5*1.5 + 0.25*0.75 + 0.125*0.25 = 2.96875;
+ // 2.0 - 0.5*1.5 + 0.25*0.75 - 0.125*0.25 = 1.40625.
+ static_assert(polyeval(T(0.5), C[0], C[1], C[2], C[3]) == T(2.96875));
+ static_assert(polyeval(T(0.5), C) == T(2.96875));
+ static_assert(loop_polyeval(T(0.5), C) == T(2.96875));
+ static_assert(estrin_polyeval(T(0.5), C) == T(2.96875));
+ static_assert(altpolyeval(T(0.5), C) == T(1.40625));
+ static_assert(loop_altpolyeval(T(0.5), C) == T(1.40625));
+#endif
+ }
+};
+
+using PolyEvalFPTypes = LIBC_NAMESPACE::testing::Types<float, double>;
+
+TYPED_TEST_SUITE(LlvmLibcPolyEvalTest, PolyEvalFPTypes);
+
+TYPED_TEST(LlvmLibcPolyEvalTest, PolyEval) { this->test_polyeval(); }
+
+TYPED_TEST(LlvmLibcPolyEvalTest, AltPolyEval) { this->test_altpolyeval(); }
+
+TYPED_TEST(LlvmLibcPolyEvalTest, EstrinPolyEval) {
+ this->test_estrin_polyeval();
+}
+
+TYPED_TEST(LlvmLibcPolyEvalTest, OrderSensitivity) {
+ this->test_order_sensitivity();
+}
+
+TYPED_TEST(LlvmLibcPolyEvalTest, ConstexprEval) { this->test_constexpr(); }
+
+TEST(LlvmLibcPolyEvalStructTest, DoubleDoubleHorner) {
+ const DoubleDouble x{0x1.3p-56, 0x1.3579bdf13579bp-2};
+ const DoubleDouble c[4] = {
+ {0x1.1p-54, 0x1.921fb54442d18p0},
+ {-0x1.5p-55, 0x1.5bf0a8b145769p-1},
+ {0x1.7p-56, 0x1.62e42fefa39efp-2},
+ {-0x1.9p-57, 0x1.71547652b82fep-3},
+ };
+
+ DoubleDouble t2 = multiply_add(x, c[3], c[2]);
+ DoubleDouble t1 = multiply_add(x, t2, c[1]);
+ DoubleDouble t0 = multiply_add(x, t1, c[0]);
+
+ DoubleDouble r1 = polyeval(x, c[0], c[1], c[2], c[3]);
+ DoubleDouble r2 = polyeval(x, c);
+ DoubleDouble r3 = loop_polyeval(x, c);
+ DoubleDouble r_slice = polyeval<1>(x, c);
+ DoubleDouble r_loop_slice = loop_polyeval<1>(x, c);
+
+ EXPECT_FP_EQ(t0.hi, r1.hi);
+ EXPECT_FP_EQ(t0.lo, r1.lo);
+ EXPECT_FP_EQ(t0.hi, r2.hi);
+ EXPECT_FP_EQ(t0.lo, r2.lo);
+ EXPECT_FP_EQ(t0.hi, r3.hi);
+ EXPECT_FP_EQ(t0.lo, r3.lo);
+ EXPECT_FP_EQ(t1.hi, r_slice.hi);
+ EXPECT_FP_EQ(t1.lo, r_slice.lo);
+ EXPECT_FP_EQ(t1.hi, r_loop_slice.hi);
+ EXPECT_FP_EQ(t1.lo, r_loop_slice.lo);
+
+ DoubleDouble s2 = sub_multiply(c[2], x, c[3]);
+ DoubleDouble s1 = sub_multiply(c[1], x, s2);
+ DoubleDouble s0 = sub_multiply(c[0], x, s1);
+
+ DoubleDouble a1 = altpolyeval(x, c[0], c[1], c[2], c[3]);
+ DoubleDouble a2 = altpolyeval(x, c);
+ DoubleDouble a3 = loop_altpolyeval(x, c);
+ EXPECT_FP_EQ(s0.hi, a1.hi);
+ EXPECT_FP_EQ(s0.lo, a1.lo);
+ EXPECT_FP_EQ(s0.hi, a2.hi);
+ EXPECT_FP_EQ(s0.lo, a2.lo);
+ EXPECT_FP_EQ(s0.hi, a3.hi);
+ EXPECT_FP_EQ(s0.lo, a3.lo);
+}
+
+TEST(LlvmLibcPolyEvalStructTest, DyadicFloatEval) {
+ using DFloat128 = DyadicFloat<128>;
+ using Mant = DFloat128::MantissaType;
+ constexpr DFloat128 X(
+ Sign::POS, -129,
+ Mant({0x1357'9bdf'2468'ace1ULL, 0x9abc'def0'1234'5678ULL}));
+ constexpr DFloat128 C[4] = {
+ DFloat128(Sign::POS, -127,
+ Mant({0xfedc'ba98'7654'3211ULL, 0xc90f'daa2'2168'c234ULL})),
+ DFloat128(Sign::POS, -128,
+ Mant({0x1122'3344'5566'7789ULL, 0xadfc'8458'a2bb'4a9aULL})),
+ DFloat128(Sign::POS, -129,
+ Mant({0x99aa'bbcc'ddee'ff01ULL, 0xb172'17f7'd1cf'79abULL})),
+ DFloat128(Sign::POS, -130,
+ Mant({0x5555'5555'5555'5557ULL, 0xb8aa'3b29'5c17'f0bbULL})),
+ };
+
+ constexpr DFloat128 T2 = multiply_add(X, C[3], C[2]);
+ constexpr DFloat128 T1 = multiply_add(X, T2, C[1]);
+ constexpr DFloat128 T0 = multiply_add(X, T1, C[0]);
+
+ constexpr DFloat128 P1 = polyeval(X, C[0], C[1], C[2], C[3]);
+ constexpr DFloat128 P2 = polyeval(X, C);
+ constexpr DFloat128 P3 = loop_polyeval(X, C);
+ static_assert(T0.mantissa == P1.mantissa && T0.exponent == P1.exponent);
+ static_assert(T0.mantissa == P2.mantissa && T0.exponent == P2.exponent);
+ static_assert(T0.mantissa == P3.mantissa && T0.exponent == P3.exponent);
+
+ constexpr DFloat128 S2 = sub_multiply(C[2], X, C[3]);
+ constexpr DFloat128 S1 = sub_multiply(C[1], X, S2);
+ constexpr DFloat128 S0 = sub_multiply(C[0], X, S1);
+
+ constexpr DFloat128 A1 = altpolyeval(X, C[0], C[1], C[2], C[3]);
+ constexpr DFloat128 A2 = altpolyeval(X, C);
+ constexpr DFloat128 A3 = loop_altpolyeval(X, C);
+ static_assert(S0.mantissa == A1.mantissa && S0.exponent == A1.exponent);
+ static_assert(S0.mantissa == A2.mantissa && S0.exponent == A2.exponent);
+ static_assert(S0.mantissa == A3.mantissa && S0.exponent == A3.exponent);
+
+ EXPECT_FP_EQ(static_cast<double>(T0), static_cast<double>(P2));
+ EXPECT_FP_EQ(static_cast<double>(S0), static_cast<double>(A2));
+}
+
+TEST(LlvmLibcPolyEvalStructTest, Frac128Eval) {
+ // Exact dyadic check in [0, 1):
+ // 0.5 + 0.25*0.25 + 0.0625*0.125 + 0.015625*0.0625 = 585 / 1024.
+ constexpr Frac128 X_EXACT({0, 0x4000'0000'0000'0000ULL}); // 0.25
+ constexpr Frac128 C_EXACT[4] = {
+ Frac128({0, 0x8000'0000'0000'0000ULL}), // 0.5
+ Frac128({0, 0x4000'0000'0000'0000ULL}), // 0.25
+ Frac128({0, 0x2000'0000'0000'0000ULL}), // 0.125
+ Frac128({0, 0x1000'0000'0000'0000ULL}), // 0.0625
+ };
+ constexpr Frac128 EXPECTED_SUM({0, (585ULL << 54)});
+ static_assert(EXPECTED_SUM == polyeval(X_EXACT, C_EXACT));
+ static_assert(EXPECTED_SUM == estrin_polyeval(X_EXACT, C_EXACT));
+
+ // Full-128-bit values where truncation differs between Horner and Estrin.
+ constexpr Frac128 X({0xffff'ffff'ffff'ffffULL, 0x5555'5555'5555'5555ULL});
+ constexpr Frac128 C[4] = {
+ Frac128({0xfedc'ba98'7654'3211ULL, 0x6a09'e667'f3bc'c908ULL}),
+ Frac128({0xffff'ffff'ffff'ffffULL, 0x3b67'ae85'84ca'a73bULL}),
+ Frac128({0xffff'ffff'ffff'ffffULL, 0x2c6e'f372'fe94'f82bULL}),
+ Frac128({0xdead'beef'cafe'bab3ULL, 0x154f'f53a'5f1d'36f1ULL}),
+ };
+
+ constexpr Frac128 H2 = multiply_add(X, C[3], C[2]);
+ constexpr Frac128 H1 = multiply_add(X, H2, C[1]);
+ constexpr Frac128 H0 = multiply_add(X, H1, C[0]);
+
+ constexpr Frac128 X2 = X * X;
+ constexpr Frac128 E_LO = multiply_add(X, C[1], C[0]);
+ constexpr Frac128 E_HI = multiply_add(X, C[3], C[2]);
+ constexpr Frac128 E0 = multiply_add(X2, E_HI, E_LO);
+
+ // Confirm Horner and Estrin produce distinct 128-bit results on C.
+ static_assert(!(H0 == E0));
+
+ static_assert(H0 == polyeval(X, C[0], C[1], C[2], C[3]));
+ static_assert(H0 == polyeval(X, C));
+ static_assert(H0 == loop_polyeval(X, C));
+ static_assert(H1 == polyeval<1>(X, C));
+ static_assert(H1 == loop_polyeval<1>(X, C));
+ static_assert(H1 == estrin_polyeval<1>(X, C));
+ static_assert(E0 == estrin_polyeval(X, C[0], C[1], C[2], C[3]));
+ static_assert(E0 == estrin_polyeval(X, C));
+
+ EXPECT_TRUE(H0 == polyeval(X, C));
+ EXPECT_TRUE(H0 == loop_polyeval(X, C));
+ EXPECT_TRUE(E0 == estrin_polyeval(X, C));
+ EXPECT_TRUE(H1 == estrin_polyeval<1>(X, C));
+
+ constexpr Frac128 A2 = sub_multiply(C[2], X, C[3]);
+ constexpr Frac128 A1 = sub_multiply(C[1], X, A2);
+ constexpr Frac128 A0 = sub_multiply(C[0], X, A1);
+ static_assert(A0 == altpolyeval(X, C[0], C[1], C[2], C[3]));
+ static_assert(A0 == altpolyeval(X, C));
+ static_assert(A0 == loop_altpolyeval(X, C));
+ EXPECT_TRUE(A0 == altpolyeval(X, C));
+ EXPECT_TRUE(A0 == loop_altpolyeval(X, C));
+}
diff --git a/utils/bazel/llvm-project-overlay/libc/src/__support/FPUtil/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/src/__support/FPUtil/BUILD.bazel
index 2b0b9e43d39ac..39ff065bfe55d 100644
--- a/utils/bazel/llvm-project-overlay/libc/src/__support/FPUtil/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/src/__support/FPUtil/BUILD.bazel
@@ -398,9 +398,11 @@ libc_support_library(
hdrs = ["polyeval.h"],
deps = [
":multiply_add",
+ "//libc/hdr/types:size_t",
"//libc/src/__support:common",
"//libc/src/__support/CPP:type_traits",
"//libc/src/__support/macros:config",
+ "//libc/src/__support/macros:optimization",
],
)
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/__support/FPUtil/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/__support/FPUtil/BUILD.bazel
index 67bad4ebfaa22..50e1b3c53562c 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/__support/FPUtil/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/__support/FPUtil/BUILD.bazel
@@ -75,6 +75,19 @@ libc_test(
],
)
+libc_test(
+ name = "polyeval_test",
+ srcs = ["polyeval_test.cpp"],
+ deps = [
+ "//libc/src/__support:frac128",
+ "//libc/src/__support/FPUtil:double_double",
+ "//libc/src/__support/FPUtil:dyadic_float",
+ "//libc/src/__support/FPUtil:multiply_add",
+ "//libc/src/__support/FPUtil:polyeval",
+ "//libc/test/UnitTest:fp_test_helpers",
+ ],
+)
+
libc_test(
name = "bfloat16_test",
srcs = ["bfloat16_test.cpp"],
More information about the llvm-branch-commits
mailing list