[llvm-branch-commits] [libc] [llvm] [libc][math] Expand polyeval.h with array, loop, and Estrin evaluation. (PR #243523)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Nov 17 16:06:19 PST 2026
================
@@ -18,69 +18,399 @@
#include "src/__support/CPP/type_traits.h"
#include "src/__support/common.h"
#include "src/__support/macros/config.h"
+#include "src/__support/macros/optimization.h"
-// Evaluate polynomial using Horner's Scheme:
-// With polyeval(x, a_0, a_1, ..., a_n) = a_n * x^n + ... + a_1 * x + a_0, we
-// evaluated it as: a_0 + x * (a_1 + x * ( ... (a_(n-1) + x * a_n) ... ) ) ).
-// We will use FMA instructions if available.
-// Example: to evaluate x^3 + 2*x^2 + 3*x + 4, call
-// polyeval( x, 4.0, 3.0, 2.0, 1.0 )
+#include <stddef.h>
namespace LIBC_NAMESPACE_DECL {
namespace fputil {
-template <typename T>
-LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(void *)), T>
-polyeval(const T &, const T &a0) {
- return a0;
+// Evaluate polynomial using Horner's scheme:
+// polyeval(x, a_0, a_1, ..., a_n) = a_0 + x * (a_1 + ... + x * a_n).
+template <typename T, typename... Ts>
+LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(double)), T>
+polyeval(T x, T a0, Ts... a) {
+ static_assert((cpp::is_same_v<T, Ts> && ...),
+ "All coefficients must have the same type T.");
+ if constexpr (sizeof...(Ts) == 0) {
+ (void)x;
+ return a0;
+ } else {
+ T r = polyeval(x, a...);
+ return multiply_add(x, r, a0);
+ }
}
-template <typename T>
-LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(void *)), T>
-polyeval(T, T a0) {
- return a0;
+template <typename T, typename... Ts>
+LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(double)), T>
+polyeval(const T &x, const T &a0, const Ts &...a) {
+ static_assert((cpp::is_same_v<T, Ts> && ...),
+ "All coefficients must have the same type T.");
+ if constexpr (sizeof...(Ts) == 0) {
+ (void)x;
+ return a0;
+ } else {
+ T r = polyeval(x, a...);
+ return multiply_add(x, r, a0);
+ }
}
+template <size_t OFFSET = 0, typename T, size_t N>
+LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(double)), T>
+polyeval(T x, const T (&a)[N]) {
+ static_assert(OFFSET < N, "OFFSET must be less than array size N.");
+ if constexpr (OFFSET + 1 == N) {
+ (void)x;
+ return a[OFFSET];
+ } else {
+ T r = polyeval<OFFSET + 1>(x, a);
+ return multiply_add(x, r, a[OFFSET]);
+ }
+}
+
+template <size_t OFFSET = 0, typename T, size_t N>
+LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(double)), T>
+polyeval(const T &x, const T (&a)[N]) {
+ static_assert(OFFSET < N, "OFFSET must be less than array size N.");
+ if constexpr (OFFSET + 1 == N) {
+ (void)x;
+ return a[OFFSET];
+ } else {
+ T r = polyeval<OFFSET + 1>(x, a);
+ return multiply_add(x, r, a[OFFSET]);
+ }
+}
+
+// Evaluate alternating polynomial:
+// altpolyeval(x, a_0, a_1, ..., a_n) = a_0 - x * a_1 + x^2 * a_2 - ...
+// + (-1)^n * x^n * a_n.
template <typename T, typename... Ts>
-LIBC_INLINE static constexpr cpp::enable_if_t<(sizeof(T) > sizeof(void *)), T>
-polyeval(const T &x, const T &a0, const Ts &...a) {
- return multiply_add(x, polyeval(x, a...), a0);
+LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(double)), T>
+altpolyeval(T x, T a0, Ts... a) {
+ static_assert((cpp::is_same_v<T, Ts> && ...),
+ "All coefficients must have the same type T.");
+ if constexpr (sizeof...(Ts) == 0) {
+ (void)x;
+ return a0;
+ } else {
+ T r = altpolyeval(x, a...);
+ return sub_multiply(a0, x, r);
+ }
}
template <typename T, typename... Ts>
-LIBC_INLINE LIBC_CONSTEXPR cpp::enable_if_t<(sizeof(T) <= sizeof(void *)), T>
-polyeval(T x, T a0, Ts... a) {
- return multiply_add(x, polyeval(x, a...), a0);
+LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(double)), T>
+altpolyeval(const T &x, const T &a0, const Ts &...a) {
+ static_assert((cpp::is_same_v<T, Ts> && ...),
+ "All coefficients must have the same type T.");
+ if constexpr (sizeof...(Ts) == 0) {
+ (void)x;
+ return a0;
+ } else {
+ T r = altpolyeval(x, a...);
+ return sub_multiply(a0, x, r);
+ }
}
-// Evaluating alternating polynomials using subtraction directly.
-// altpolyeval(x, a_0, a_1, ..., a_n) = a_0 - x * a_1 + x^2 * a_2 - ... +
-// + (-1)^n x_n * a_n.
-template <typename T>
-LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(void *)), T>
-altpolyeval(const T &, const T &a0) {
- return a0;
+template <size_t OFFSET = 0, typename T, size_t N>
+LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(double)), T>
+altpolyeval(T x, const T (&a)[N]) {
+ static_assert(OFFSET < N, "OFFSET must be less than array size N.");
+ if constexpr (OFFSET + 1 == N) {
+ (void)x;
+ return a[OFFSET];
+ } else {
+ T r = altpolyeval<OFFSET + 1>(x, a);
+ return sub_multiply(a[OFFSET], x, r);
+ }
+}
+
+template <size_t OFFSET = 0, typename T, size_t N>
+LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(double)), T>
+altpolyeval(const T &x, const T (&a)[N]) {
+ static_assert(OFFSET < N, "OFFSET must be less than array size N.");
+ if constexpr (OFFSET + 1 == N) {
+ (void)x;
+ return a[OFFSET];
+ } else {
+ T r = altpolyeval<OFFSET + 1>(x, a);
+ return sub_multiply(a[OFFSET], x, r);
+ }
+}
+
+// Evaluate polynomial using a non-unrolled Horner loop:
+// loop_polyeval<OFFSET>(x, a) = a[OFFSET] + x * (a[OFFSET + 1] + ...
+// + x * a[N - 1]).
+template <size_t OFFSET = 0, typename T, size_t N>
+LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(double)), T>
+loop_polyeval(T x, const T (&a)[N]) {
+ static_assert(OFFSET < N, "OFFSET must be less than array size N.");
+ T r = a[N - 1];
+ if constexpr (OFFSET + 1 < N) {
+ LIBC_LOOP_NOUNROLL
+ for (size_t i = N - 1; i > OFFSET; --i)
+ r = multiply_add(x, r, a[i - 1]);
+ } else {
+ (void)x;
+ }
+ return r;
+}
+
+template <size_t OFFSET = 0, typename T, size_t N>
+LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(double)), T>
+loop_polyeval(const T &x, const T (&a)[N]) {
+ static_assert(OFFSET < N, "OFFSET must be less than array size N.");
+ T r = a[N - 1];
+ if constexpr (OFFSET + 1 < N) {
+ LIBC_LOOP_NOUNROLL
+ for (size_t i = N - 1; i > OFFSET; --i)
+ r = multiply_add(x, r, a[i - 1]);
+ } else {
+ (void)x;
+ }
+ return r;
}
+// Evaluate alternating polynomial using a non-unrolled Horner loop:
+// loop_altpolyeval<OFFSET>(x, a) = a[OFFSET] - x * a[OFFSET + 1] + ...
+// + (-1)^(N - 1 - OFFSET) * x^(N - 1 - OFFSET) * a[N - 1].
+template <size_t OFFSET = 0, typename T, size_t N>
+LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(double)), T>
+loop_altpolyeval(T x, const T (&a)[N]) {
+ static_assert(OFFSET < N, "OFFSET must be less than array size N.");
+ T r = a[N - 1];
+ if constexpr (OFFSET + 1 < N) {
+ LIBC_LOOP_NOUNROLL
+ for (size_t i = N - 1; i > OFFSET; --i)
+ r = sub_multiply(a[i - 1], x, r);
+ } else {
+ (void)x;
+ }
+ return r;
+}
+
+template <size_t OFFSET = 0, typename T, size_t N>
+LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) > sizeof(double)), T>
+loop_altpolyeval(const T &x, const T (&a)[N]) {
+ static_assert(OFFSET < N, "OFFSET must be less than array size N.");
+ T r = a[N - 1];
+ if constexpr (OFFSET + 1 < N) {
+ LIBC_LOOP_NOUNROLL
+ for (size_t i = N - 1; i > OFFSET; --i)
+ r = sub_multiply(a[i - 1], x, r);
+ } else {
+ (void)x;
+ }
+ return r;
+}
+
+namespace internal {
+
template <typename T>
-LIBC_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(void *)), T>
-altpolyeval(T, T a0) {
- return a0;
+using ValOrConstRef =
+ cpp::conditional_t<(sizeof(T) > sizeof(double)), const T &, T>;
+
+// For len >= 2, let 2^h < len <= 2^(h + 1), so tree depth is h + 1 and each
+// subtree has capacity at most 2^h. Return the smallest k such that
+// 2^k >= len - 2^h, giving a right-heavy split:
+// LOW = 2^k <= 2^h, 1 <= HI = len - 2^k <= 2^h.
+// Examples:
+// len = 2, 3 -> k = 0 (1 + 1, 1 + 2)
+// len = 4 -> k = 1 (2 + 2)
+// len = 5 -> k = 0 (1 + 4)
+// len = 6 -> k = 1 (2 + 4)
+// len = 7, 8 -> k = 2 (4 + 3, 4 + 4)
+// len = 9 -> k = 0 (1 + 8)
+// len = 10 -> k = 1 (2 + 8)
+// len = 11, 12 -> k = 2 (4 + 7, 4 + 8)
+LIBC_INLINE constexpr size_t estrin_split_log2(size_t len) {
+ size_t h = 0;
+ while ((size_t(1) << (h + 1)) < len)
+ ++h;
+ size_t rem = len - (size_t(1) << h);
+ size_t k = 0;
+ while ((size_t(1) << k) < rem)
+ ++k;
+ return k;
+}
+
+// Return the largest dyadic power index K needed across the entire right-heavy
+// Estrin tree of length len (for len >= 2). Powers up to x^(2^K) can form a
+// full high subtree of length 2^(K + 1) plus a low subtree of length 2^K,
+// covering len <= 3 * 2^K:
+// len = 2, 3 -> K = 0 (x)
+// len = 4..6 -> K = 1 (x, x^2)
+// len = 7..12 -> K = 2 (x, x^2, x^4)
+// len = 13..24 -> K = 3 (x, x^2, x^4, x^8)
+LIBC_INLINE constexpr size_t estrin_max_power_log2(size_t len) {
+ size_t k = 0;
+ while ((size_t(3) << k) < len)
+ ++k;
+ return k;
+}
+
+// Precompute dyadic powers x^(2^0), x^(2^1), ..., x^(2^K) once by repeated
+// squaring so that all subtrees share x^2, x^4, ..., x^(2^K). Members are
+// initialized in declaration order (`prev` before `x_pow`), computing:
+// EstrinPows<T, 0>: x_pow = x
+// EstrinPows<T, K>: prev = EstrinPows<T, K - 1>(x),
+// x_pow = prev.x_pow * prev.x_pow = x^(2^K)
+// pows.get<I>() returns x^(2^I) for 0 <= I <= K.
+template <typename T, size_t K> struct EstrinPows {
+ EstrinPows<T, K - 1> prev;
+ T x_pow;
+
+ LIBC_ALWAYS_INLINE constexpr explicit EstrinPows(ValOrConstRef<T> x)
+ : prev(x), x_pow(prev.x_pow * prev.x_pow) {}
+
+ template <size_t I>
+ LIBC_ALWAYS_INLINE constexpr ValOrConstRef<T> get() const {
+ if constexpr (I == K)
+ return x_pow;
+ else
+ return prev.template get<I>();
+ }
+};
+
+template <typename T> struct EstrinPows<T, 0> {
+ T x_pow;
+
+ LIBC_ALWAYS_INLINE constexpr explicit EstrinPows(ValOrConstRef<T> x)
+ : x_pow(x) {}
+
+ template <size_t I>
+ LIBC_ALWAYS_INLINE constexpr ValOrConstRef<T> get() const {
+ static_assert(I == 0, "Invalid Estrin power index.");
+ return x_pow;
+ }
+};
+
+// Recursively evaluate P(x) = sum_{i = 0}^{LEN - 1} a[OFFSET + i] * x^i using
+// a right-heavy Estrin binary tree of minimum depth ceil(log2(LEN)):
+// - LEN == 1: a[OFFSET]
+// - LEN == 2: multiply_add(x, a[OFFSET + 1], a[OFFSET])
+// = a[OFFSET] + x * a[OFFSET + 1]
+// - LEN >= 3: let K = estrin_split_log2(LEN) and LOW = 2^K. Split into:
+// lo = estrin_eval_tree<OFFSET, LOW>(pows, a)
+// = sum_{i = 0}^{LOW - 1} a[OFFSET + i] * x^i
+// hi = estrin_eval_tree<OFFSET + LOW, LEN - LOW>(pows, a)
+// = sum_{i = 0}^{LEN - LOW - 1} a[OFFSET + LOW + i] * x^i
+// and combine via multiply_add(x^(2^K), hi, lo) = lo + x^(2^K) * hi.
+//
+// Placing the larger subtree in `hi` minimizes both the highest required power
+// of x and the number of roundings on a[OFFSET] when |x| << 1, while keeping
+// tree depth ceil(log2(LEN)). With x2 = x^2 and x4 = x^4, each (u + p * v)
+// below is evaluated as multiply_add(p, v, u):
+// LEN = 3: a0 + x * (a1 + x * a2)
+// LEN = 4: (a0 + x * a1) + x2 * (a2 + x * a3)
+// LEN = 5: a0 + x * ((a1 + x * a2) + x2 * (a3 + x * a4))
+// LEN = 6: (a0 + x * a1) + x2 * ((a2 + x * a3) + x2 * (a4 + x * a5))
+// LEN = 7: ((a0 + x * a1) + x2 * (a2 + x * a3))
+// + x4 * (a4 + x * (a5 + x * a6))
+// LEN = 8: ((a0 + x * a1) + x2 * (a2 + x * a3))
+// + x4 * ((a4 + x * a5) + x2 * (a6 + x * a7))
+template <size_t OFFSET, size_t LEN, typename T, size_t N, size_t MAX_K>
+LIBC_ALWAYS_INLINE constexpr cpp::enable_if_t<(sizeof(T) <= sizeof(double)), T>
+estrin_eval_tree(const EstrinPows<T, MAX_K> &pows, const T (&a)[N]) {
----------------
lntue wrote:
Most of the math look-up tables are constant C arrays, which will not bind to `const cpp::array<>` or `const cpp::array<>&`. And `cpp::array<> a` can be used with `a.Data`, so that we don't have to duplicate all the overloads for `const cpp::array<>&`.
https://github.com/llvm/llvm-project/pull/243523
More information about the llvm-branch-commits
mailing list