[libc-commits] [libc] ef1e579 - [libc][mathvec] Vectorise cosf (#224290)
via libc-commits
libc-commits at lists.llvm.org
Tue Sep 29 11:49:15 PDT 2026
Author: DylanFleming-arm
Date: 2026-09-29T18:49:08Z
New Revision: ef1e579be4898441f4b340bf7e416397cc1d8f4f
URL: https://github.com/llvm/llvm-project/commit/ef1e579be4898441f4b340bf7e416397cc1d8f4f
DIFF: https://github.com/llvm/llvm-project/commit/ef1e579be4898441f4b340bf7e416397cc1d8f4f.diff
LOG: [libc][mathvec] Vectorise cosf (#224290)
Replaces loop over scalar cosf with a fully vectorised implementation.
Replaces sinf and cosf scalar reference with a forced accurate double-eval.
Added:
Modified:
libc/src/__support/mathvec/CMakeLists.txt
libc/src/__support/mathvec/cosf.h
libc/src/__support/mathvec/trig_reductionf.h
libc/src/__support/mathvec/trig_reductionf_nofma.h
libc/test/UnitTest/SIMDMatcher.h
libc/test/src/mathvec/UnitTestWrappers.h
libc/test/src/mathvec/cosf_test.cpp
libc/test/src/mathvec/exhaustive/cosf_test.cpp
libc/test/src/mathvec/exhaustive/exhaustive_test.h
libc/test/src/mathvec/exhaustive/sinf_test.cpp
libc/test/src/mathvec/sinf_test.cpp
Removed:
################################################################################
diff --git a/libc/src/__support/mathvec/CMakeLists.txt b/libc/src/__support/mathvec/CMakeLists.txt
index 431ad79f10501..dba7b6baba3d6 100644
--- a/libc/src/__support/mathvec/CMakeLists.txt
+++ b/libc/src/__support/mathvec/CMakeLists.txt
@@ -87,7 +87,8 @@ add_header_library(
cosf.h
DEPENDS
libc.src.__support.CPP.simd
- libc.src.__support.math.cosf
+ libc.src.__support.FPUtil.fp_bits
+ libc.src.__support.macros.properties.cpu_features
)
add_header_library(
diff --git a/libc/src/__support/mathvec/cosf.h b/libc/src/__support/mathvec/cosf.h
index b6ea0c848afd7..0d3cbe2922898 100644
--- a/libc/src/__support/mathvec/cosf.h
+++ b/libc/src/__support/mathvec/cosf.h
@@ -15,18 +15,132 @@
#define LLVM_LIBC_SRC___SUPPORT_MATHVEC_COSF_H
#include "src/__support/CPP/simd.h"
-#define LIBC_MATH_HAS_NO_ERRNO
-#define LIBC_MATH_HAS_NO_EXCEPT
-#define LIBC_MATH_HAS_ASSUME_ROUND_NEAREST_ONLY
-#include "src/__support/math/cosf.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/macros/properties/cpu_features.h"
+
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
+#include "src/__support/mathvec/trig_reductionf.h"
+#else
+#include "src/__support/mathvec/trig_reductionf_nofma.h"
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
namespace LIBC_NAMESPACE_DECL {
namespace mathvec {
+template <size_t N>
+LIBC_INLINE static cpp::simd<double, N> cospif_poly(cpp::simd<double, N> r) {
+ // Approximate cos(pi * r) as (1/4 - r^2) * P(r^2) for |r| <= 0.5.
+ // These coefficients aren't produced directly via sollya, but rather
+ // are fine-tuned by iterative adjustment to remove hard to round cases.
+ // TODO: Create a tool to deterministically reproduce these coefficients.
+ // see https://github.com/llvm/llvm-project/issues/220984
+ constexpr cpp::simd<double, N> c0 = 0x1p2;
+ constexpr cpp::simd<double, N> c1 = -0x1.de9e64df22f07p1;
+ constexpr cpp::simd<double, N> c2 = 0x1.472be1223eeadp0;
+ constexpr cpp::simd<double, N> c3 = -0x1.d4fcd82b511ebp-3;
+ constexpr cpp::simd<double, N> c4 = 0x1.9f05c866286ddp-6;
+ constexpr cpp::simd<double, N> c5 = -0x1.f308c07837812p-10;
+ constexpr cpp::simd<double, N> c6 = 0x1.b22004be59c73p-14;
+ constexpr cpp::simd<double, N> c7 = -0x1.14bd54572305fp-18;
+ constexpr cpp::simd<double, N> f = 0.25;
+
+ cpp::simd<double, N> r2 = r * r;
+ cpp::simd<double, N> r4 = r2 * r2;
+ cpp::simd<double, N> p01 = cpp::multiply_add(r2, c1, c0);
+ cpp::simd<double, N> p23 = cpp::multiply_add(r2, c3, c2);
+ cpp::simd<double, N> p45 = cpp::multiply_add(r2, c5, c4);
+ cpp::simd<double, N> p67 = cpp::multiply_add(r2, c7, c6);
+ cpp::simd<double, N> p47 = cpp::multiply_add(r4, p67, p45);
+ cpp::simd<double, N> p27 = cpp::multiply_add(r4, p47, p23);
+ cpp::simd<double, N> p07 = cpp::multiply_add(r4, p27, p01);
+
+ cpp::simd<double, N> factor = cpp::multiply_add(-r, r, f);
+
+ return factor * p07;
+}
+
+// Correct specific cases which aren't able to correctly round from the normal
+// codepath.
+template <size_t N>
+LIBC_INLINE static cpp::simd<float, N>
+repair_hard_to_round(cpp::simd<float, N> x, cpp::simd<float, N> y) {
+ y = (x == 0x1.fcd9eep+38f) ? 0x1.3371d2p-17f : y;
+ y = (x == 0x1.3170f0p+63f) ? 0x1.fe2976p-1f : y;
+ y = (x == 0x1.15313ep+69f) ? 0x1.f52484p-20f : y;
+
+#ifndef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
+ y = (x == 0x1.8db252p+25f) ? -0x1.527a0ap-5f : y;
+ y = (x == 0x1.aff73cp+32f) ? -0x1.d59e7ap-23f : y;
+ y = (x == 0x1.47d0fep+34f) ? -0x1.149dbp-29f : y;
+ y = (x == 0x1.455500p+51f) ? 0x1.115d7ep-1f : y;
+ y = (x == 0x1.407c74p+100f) ? 0x1.a4a122p-19f : y;
+#endif
+
+ return y;
+}
+
+// Due to specific lane correction being expensive, we can perform a fast
+// estimated check to determine if we should branch to specific hard to round
+// case correction. Will return true for all hard to round cases, but can
+// produce false positives. False positive rate in [0x1.90bdfap+20f, inf]:
+// With FMA: 52/901,226,755 ~= 1/2^24
+// Without FMA: 7,040,869/901,226,755 ~= 1/2^7
+template <size_t N>
+LIBC_INLINE static cpp::simd<bool, N>
+is_maybe_hard_to_round(cpp::simd<float, N> x) {
+ cpp::simd<uint32_t, N> x_bits = cpp::bit_cast<cpp::simd<uint32_t>>(x);
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
+ return ((x_bits * 0xd6c4de5f) & 0xefffda56) == 0x46048000;
+#else
+ return ((x_bits * 0x30d04f5f) & 0x8b002240) == 0x8002000;
+#endif
+}
+
template <size_t N>
LIBC_INLINE cpp::simd<float, N> cosf(cpp::simd<float, N> x) {
- return cpp::map(x, [](float a) { return math::cosf(a); });
+ using FPBits = typename fputil::FPBits<float>;
+ cpp::simd<double, N> x_d = cpp::simd_cast<double>(x);
+ cpp::simd<float, N> ax = cpp::abs(x);
+
+ // If all lanes are below pi/2, we can skip the reduction entirely.
+ cpp::simd<bool, N> is_small = ax <= 0x1.921fb6p+0f;
+ if (cpp::all_of(is_small)) {
+ constexpr cpp::simd<double, N> inv_pi = 0x1.45f306dc9c883p-2;
+ return cpp::simd_cast<float>(cospif_poly(x_d * inv_pi));
+ }
+
+ // Computes the main reduction pass
+ Reduction<N> reduce = fast_reduction(x_d);
+
+ // Large inputs require a more involved reduction, as well as inf handling.
+ cpp::simd<bool, N> has_large_reduction = ax > 0x1.90bdfap+20f;
+ if (LIBC_UNLIKELY(cpp::any_of(has_large_reduction))) {
+ cpp::simd<bool, N> is_finite = ax < FPBits::inf().get_val();
+ Reduction<N> large_reduce = large_reduction(x_d);
+ reduce.r = has_large_reduction ? large_reduce.r : reduce.r;
+ reduce.r = is_finite ? reduce.r : FPBits::quiet_nan().get_val();
+ reduce.odd = has_large_reduction ? large_reduce.odd : reduce.odd;
+ }
+
+ // Both reduction paths feed into a single polynomial evaluation + sign
+ // correction.
+ cpp::simd<float, N> poly = cpp::simd_cast<float>(cospif_poly(reduce.r));
+ cpp::simd<uint32_t, N> sign = cpp::simd_cast<uint32_t>(reduce.odd) << 31;
+
+ cpp::simd<float, N> y = cpp::bit_cast<cpp::simd<float>>(
+ cpp::bit_cast<cpp::simd<uint32_t>>(poly) ^ sign);
+
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+#ifndef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
+ y = (ax == 0x1.d2eb54p+10) ? -0x1.633eb4p-13 : y;
+#endif
+ if (LIBC_UNLIKELY(cpp::any_of(has_large_reduction))) {
+ if (LIBC_UNLIKELY(cpp::any_of(is_maybe_hard_to_round(ax))))
+ return repair_hard_to_round(ax, y);
+ }
+#endif // LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+ return y;
}
} // namespace mathvec
diff --git a/libc/src/__support/mathvec/trig_reductionf.h b/libc/src/__support/mathvec/trig_reductionf.h
index dac2506683400..5df61fe65d1f2 100644
--- a/libc/src/__support/mathvec/trig_reductionf.h
+++ b/libc/src/__support/mathvec/trig_reductionf.h
@@ -40,19 +40,19 @@ LIBC_INLINE static Reduction<N> fast_reduction(cpp::simd<double, N> x) {
cpp::simd<double, N> t = shift - z;
// r = x/pi - k
- cpp::simd<double, N> r;
- r = cpp::multiply_add(x, inv_pi, t);
+ cpp::simd<double, N> r = cpp::multiply_add(x, inv_pi, t);
r = cpp::multiply_add(x, inv_pi_tail, r);
return {r, cpp::bit_cast<cpp::simd<int64_t, N>>(z)};
}
-// Two-double expansions of 2^(8*q) / pi reduced modulo an even integer,
-// q = 3..12. Padded to a 16 length array for easier access.
+// Two-double expansions of 2^(8*q) / pi reduced modulo an even integer.
+// Entries 0..14 correspond to q = 0..14, and entry 15 corresponds to q = -1
+// after masking. Entry 13 and 14 act as padding for inactive lanes.
LIBC_INLINE_VAR constexpr double INV_PI_HI[16] = {
- 0,
- 0,
- 0,
+ 0x1.45f306dc9c883p-2,
+ -0x1.067c91b1bbeadp-1,
+ 0x1.836e4e44152ap-1,
-0x1.236377d5ac07bp-2,
-0x1.b1bbead603d8bp-1,
-0x1.bbead603d8a83p-1,
@@ -65,13 +65,13 @@ LIBC_INLINE_VAR constexpr double INV_PI_HI[16] = {
0x1.f534ddc0db629p-1,
0,
0,
- 0,
+ 0x1.45f306dc9c883p-10,
};
LIBC_INLINE_VAR constexpr double INV_PI_LO[16] = {
- 0,
- 0,
- 0,
+ -0x1.6b01ec55ff4d6p-56,
+ -0x1.80f62a0b82b2dp-55,
+ -0x1.ec54170565912p-56,
-0x1.505c1596447e5p-58,
0x1.f47d4d377036ep-55,
0x1.f534ddc0db629p-57,
@@ -84,11 +84,12 @@ LIBC_INLINE_VAR constexpr double INV_PI_LO[16] = {
0x1.664f10e4107f9p-55,
0,
0,
- 0,
+ -0x1.6b01e7abff3d6p-64,
};
// Reduces non-negative large finite inputs x >= 0x1p49.
// Decomposes x / pi into k + r, with k as an integer and |r| <= 0.5.
+// Based on a paper written by Tue Ly, see https://arxiv.org/abs/2609.35015
template <size_t N>
LIBC_INLINE static Reduction<N> large_reduction(cpp::simd<double, N> x) {
constexpr cpp::simd<double, N> shift = 0x1.8p52;
@@ -100,9 +101,8 @@ LIBC_INLINE static Reduction<N> large_reduction(cpp::simd<double, N> x) {
cpp::simd<int64_t, N> q =
cpp::simd_cast<int64_t>(ix >> 55) - cpp::simd<int64_t, N>(131);
- // While sufficiently large x will always produce q within [3, 12],
- // not all input lanes are guaranteed to require the large reduction,
- // so we mask with 15 to keep all values within bounds.
+ // For float inputs requiring large reduction, q is in [-1, 12]. Masking
+ // maps q = -1 to entry 15 and keeps inactive lanes within bounds.
cpp::simd<int64_t, N> idx = q & cpp::simd<int64_t, N>(15);
cpp::simd<double, N> c_hi =
cpp::gather<cpp::simd<double, N>>(true, idx, INV_PI_HI);
diff --git a/libc/src/__support/mathvec/trig_reductionf_nofma.h b/libc/src/__support/mathvec/trig_reductionf_nofma.h
index ac2f029922c5b..2d63e9c552fa8 100644
--- a/libc/src/__support/mathvec/trig_reductionf_nofma.h
+++ b/libc/src/__support/mathvec/trig_reductionf_nofma.h
@@ -55,12 +55,15 @@ LIBC_INLINE static Reduction<N> fast_reduction(cpp::simd<double, N> x) {
return {r, cpp::bit_cast<cpp::simd<int64_t, N>>(z)};
}
-// Three-double expansions of 2^(8*q) / pi, reduced modulo an even integer,
-// q = 3..12. Padded to 16 length arrays for safe masked indexing.
+// Three-double expansions of 2^(8*q) / pi reduced modulo an even integer.
+// Entries 0..14 correspond to q = 0..14, and entry 15 corresponds to q = -1
+// after masking. Entry 13 and 14 act as padding for inactive lanes.
+// The first two parts have 29 significant bits, such that the fp64 product
+// is exact when multiplied by an fp32 input.
LIBC_INLINE_VAR constexpr double INV_PI_HI[16] = {
- 0,
- 0,
- 0,
+ 0x1.45f306ep-2,
+ -0x1.067c91bp-1,
+ 0x1.836e4e4p-1,
-0x1.236377d000000p-2,
-0x1.b1bbead000000p-1,
-0x1.bbead60000000p-1,
@@ -73,12 +76,13 @@ LIBC_INLINE_VAR constexpr double INV_PI_HI[16] = {
0x1.f534ddc000000p-1,
0,
0,
+ 0x1.45f306ep-10,
};
LIBC_INLINE_VAR constexpr double INV_PI_MID[16] = {
- 0,
- 0,
- 0,
+ -0x1.b1bbeadp-33,
+ -0x1.bbead6p-33,
+ 0x1.054a7f1p-31,
-0x1.6b01ec5000000p-32,
-0x1.80f62a1000000p-31,
-0x1.ec54170000000p-32,
@@ -91,12 +95,13 @@ LIBC_INLINE_VAR constexpr double INV_PI_MID[16] = {
0x1.b6c52b3000000p-34,
0,
0,
+ -0x1.b1bbeadp-41,
};
LIBC_INLINE_VAR constexpr double INV_PI_LO[16] = {
- 0,
- 0,
- 0,
+ -0x1.80f62a0b82b2dp-63,
+ -0x1.ec54170565912p-64,
+ -0x1.8a82e0acb223fp-61,
-0x1.05c1596447e50p-62,
0x1.1f534ddc0db80p-61,
-0x1.596447e493ae0p-62,
@@ -109,10 +114,12 @@ LIBC_INLINE_VAR constexpr double INV_PI_LO[16] = {
0x1.3c439041fe400p-65,
0,
0,
+ -0x1.80f62a0b82b2dp-71,
};
// Reduces non-negative large finite inputs x >= 0x1p49.
// Decomposes x / pi into k + r, with k as an integer and |r| <= 0.5.
+// Based on a paper written by Tue Ly, see https://arxiv.org/abs/2609.35015
template <size_t N>
LIBC_INLINE static Reduction<N> large_reduction(cpp::simd<double, N> x) {
constexpr cpp::simd<double, N> shift = 0x1.8p52;
@@ -124,9 +131,8 @@ LIBC_INLINE static Reduction<N> large_reduction(cpp::simd<double, N> x) {
cpp::simd<int64_t, N> q =
cpp::simd_cast<int64_t>(ix >> 55) - cpp::simd<int64_t, N>(131);
- // While sufficiently large x will always produce q within [3, 12],
- // not all input lanes are guaranteed to require the large reduction,
- // so we mask with 15 to keep all values within bounds.
+ // For float inputs requiring large reduction, q is in [-1, 12]. Masking
+ // maps q = -1 to entry 15 and keeps inactive lanes within bounds.
cpp::simd<int64_t, N> idx = q & cpp::simd<int64_t, N>(15);
cpp::simd<double, N> c_hi =
cpp::gather<cpp::simd<double, N>>(true, idx, INV_PI_HI);
diff --git a/libc/test/UnitTest/SIMDMatcher.h b/libc/test/UnitTest/SIMDMatcher.h
index 542d311ddbd89..a197fbfddcd88 100644
--- a/libc/test/UnitTest/SIMDMatcher.h
+++ b/libc/test/UnitTest/SIMDMatcher.h
@@ -9,6 +9,7 @@
#ifndef LLVM_LIBC_TEST_UNITTEST_SIMDMATCHER_H
#define LLVM_LIBC_TEST_UNITTEST_SIMDMATCHER_H
+#include "hdr/stdint_proxy.h"
#include "src/__support/FPUtil/FPBits.h"
#include "src/__support/macros/config.h"
#include "src/__support/macros/properties/architectures.h"
@@ -17,12 +18,56 @@
#include "hdr/math_macros.h"
-#define EXPECT_SIMD_EQ(REF, RES) \
+namespace LIBC_NAMESPACE_DECL {
+namespace testing {
+
+template <typename T>
+inline bool within_ulp_tolerance(T expected, T actual, uint64_t tolerance) {
+ fputil::FPBits<T> expected_bits(expected), actual_bits(actual);
+
+ // Handle inf and nan cases.
+ if (expected_bits.is_inf() || expected_bits.is_inf())
+ return expected_bits.is_inf() && expected_bits.is_inf();
+
+ if (expected_bits.is_nan() || actual_bits.is_nan())
+ return expected_bits.is_nan() && actual_bits.is_nan();
+
+ // Find the absolute
diff erence of the input bits
+ auto expected_uint = expected_bits.uintval();
+ auto actual_uint = actual_bits.uintval();
+ auto
diff erence = expected_uint > actual_uint ? expected_uint - actual_uint
+ : actual_uint - expected_uint;
+
+ // Allow results that are within tolerance.
+ return
diff erence <= tolerance;
+}
+
+} // namespace testing
+} // namespace LIBC_NAMESPACE_DECL
+
+#define EXPECT_SIMD_EQ_EXACT(REF, RES) \
for (size_t i = 0; \
i < LIBC_NAMESPACE::cpp::internal::native_vector_size<float>; i++) { \
EXPECT_FP_EQ(REF[i], RES[i]); \
}
+#define EXPECT_SIMD_EQ_TOL(REF, RES, TOL) \
+ do { \
+ auto simd_ref = (REF); \
+ auto simd_res = (RES); \
+ for (size_t i = 0; \
+ i < LIBC_NAMESPACE::cpp::internal::native_vector_size<float>; i++) { \
+ EXPECT_TRUE(LIBC_NAMESPACE::testing::within_ulp_tolerance( \
+ simd_ref[i], simd_res[i], (TOL))); \
+ } \
+ } while (0)
+
+#define EXPECT_SIMD_EQ_SELECT(_1, _2, _3, NAME, ...) NAME
+#define EXPECT_SIMD_EQ(...) \
+ EXPECT_SIMD_EQ_SELECT(__VA_ARGS__, EXPECT_SIMD_EQ_TOL, EXPECT_SIMD_EQ_EXACT, \
+ unused) \
+ (__VA_ARGS__)
+
#define EXPECT_SIMD_EQ_WITH_EXCEPTION(REF, RES, EXCEPTION) \
for (size_t i = 0; \
i < LIBC_NAMESPACE::cpp::internal::native_vector_size<float>; i++) { \
diff --git a/libc/test/src/mathvec/UnitTestWrappers.h b/libc/test/src/mathvec/UnitTestWrappers.h
index 3d9af60130efa..aadd918a19003 100644
--- a/libc/test/src/mathvec/UnitTestWrappers.h
+++ b/libc/test/src/mathvec/UnitTestWrappers.h
@@ -97,6 +97,20 @@ LIBC_INLINE typename Op::VectorType wrap_vector(typename Op::ScalarType x) {
EXPECT_SIMD_EQ(wrap_ref<Op>((x), 1.0), wrap_vector<Op>((x), 1.0)); \
} while (0)
+// Use the same control inputs with an allowed
diff erence in output ULPs.
+#define TEST_VARIED_CASES_TOL(x, Op, TOL) \
+ do { \
+ EXPECT_SIMD_EQ(wrap_ref<Op>((x), -(x)), wrap_vector<Op>((x), -(x)), TOL); \
+ EXPECT_SIMD_EQ(wrap_ref<Op>(-(x), (x)), wrap_vector<Op>(-(x), (x)), TOL); \
+ EXPECT_SIMD_EQ(wrap_ref<Op>((x), aNaN), wrap_vector<Op>((x), aNaN), TOL); \
+ EXPECT_SIMD_EQ(wrap_ref<Op>((x), inf), wrap_vector<Op>((x), inf), TOL); \
+ EXPECT_SIMD_EQ(wrap_ref<Op>((x), neg_inf), wrap_vector<Op>((x), neg_inf), \
+ TOL); \
+ EXPECT_SIMD_EQ(wrap_ref<Op>((x), 0.0), wrap_vector<Op>((x), 0.0), TOL); \
+ EXPECT_SIMD_EQ(wrap_ref<Op>((x), -0.0), wrap_vector<Op>((x), -0.0), TOL); \
+ EXPECT_SIMD_EQ(wrap_ref<Op>((x), 1.0), wrap_vector<Op>((x), 1.0), TOL); \
+ } while (0)
+
// A helper macro to test a full range of float values, from 0 to 0x7f800000.
// Negative values are tested via the TEST_VARIED_CASES macro.
// The number of values to test is controlled by the LIBC_TEST_FLOAT_RANGE_COUNT
@@ -104,7 +118,7 @@ LIBC_INLINE typename Op::VectorType wrap_vector(typename Op::ScalarType x) {
#define TEST_MATHVEC_FLOAT_RANGE(Op) \
do { \
constexpr uint32_t COUNT = LIBC_TEST_FLOAT_RANGE_COUNT; \
- constexpr uint32_t RANGE = 0x7f800000U; \
+ constexpr uint32_t RANGE = 0x7f80'0000U; \
constexpr uint32_t STEP = (RANGE / COUNT) > 0 ? (RANGE / COUNT) : 1; \
for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) { \
float x = FPBits(v).get_val(); \
@@ -112,6 +126,18 @@ LIBC_INLINE typename Op::VectorType wrap_vector(typename Op::ScalarType x) {
} \
} while (0)
+// Repeat the range test with an allowed
diff erence in output ULPs.
+#define TEST_MATHVEC_FLOAT_RANGE_TOL(Op, TOL) \
+ do { \
+ constexpr uint32_t COUNT = LIBC_TEST_FLOAT_RANGE_COUNT; \
+ constexpr uint32_t RANGE = 0x7f80'0000U; \
+ constexpr uint32_t STEP = (RANGE / COUNT) > 0 ? (RANGE / COUNT) : 1; \
+ for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) { \
+ float x = FPBits(v).get_val(); \
+ TEST_VARIED_CASES_TOL(x, Op, TOL); \
+ } \
+ } while (0)
+
} // namespace mathvec
} // namespace testing
} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/mathvec/cosf_test.cpp b/libc/test/src/mathvec/cosf_test.cpp
index 3ea41e7fd099f..f33d9200eb6b6 100644
--- a/libc/test/src/mathvec/cosf_test.cpp
+++ b/libc/test/src/mathvec/cosf_test.cpp
@@ -14,7 +14,15 @@
#include "hdr/math_macros.h"
#include "src/__support/CPP/simd.h"
#include "src/__support/FPUtil/FPBits.h"
-#include "src/math/cosf.h"
+#include "src/__support/macros/optimization.h"
+#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+#define MATHVEC_TOL 1
+#else
+#define MATHVEC_TOL 0
+#endif
+// Keep the scalar reference correctly rounded in every build configuration.
+#undef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+#include "src/__support/math/cosf_double_eval.h"
#include "src/mathvec/cosf.h"
#include "test/UnitTest/SIMDMatcher.h"
#include "test/UnitTest/Test.h"
@@ -25,9 +33,8 @@
using LlvmLibcVecCosfTest = LIBC_NAMESPACE::testing::FPTest<float>;
-using CosfOp =
- LIBC_NAMESPACE::testing::mathvec::UnaryOp<float, LIBC_NAMESPACE::cosf,
- LIBC_NAMESPACE::cosf>;
+using CosfOp = LIBC_NAMESPACE::testing::mathvec::UnaryOp<
+ float, LIBC_NAMESPACE::math::double_eval::cosf, LIBC_NAMESPACE::cosf>;
using LIBC_NAMESPACE::cpp::splat;
using LIBC_NAMESPACE::testing::SDCOMP26094_VALUES;
using LIBC_NAMESPACE::testing::mathvec::wrap_ref;
@@ -50,7 +57,8 @@ TEST_F(LlvmLibcVecCosfTest, SpecialNumbers) {
TEST_F(LlvmLibcVecCosfTest, SDCOMP_26094) {
for (uint32_t v : SDCOMP26094_VALUES) {
float x = FPBits((v)).get_val();
- EXPECT_SIMD_EQ(wrap_ref<CosfOp>(x, -x), wrap_vector<CosfOp>(x, -x));
+ EXPECT_SIMD_EQ(wrap_ref<CosfOp>(x, -x), wrap_vector<CosfOp>(x, -x),
+ MATHVEC_TOL);
}
}
@@ -103,8 +111,11 @@ TEST_F(LlvmLibcVecCosfTest, SpecificBitPatterns) {
for (int i = 0; i < N; ++i) {
float x = FPBits(INPUTS[i]).get_val();
- EXPECT_SIMD_EQ(wrap_ref<CosfOp>(x, -x), wrap_vector<CosfOp>(x, -x));
+ EXPECT_SIMD_EQ(wrap_ref<CosfOp>(x, -x), wrap_vector<CosfOp>(x, -x),
+ MATHVEC_TOL);
}
}
-TEST_F(LlvmLibcVecCosfTest, InFloatRange) { TEST_MATHVEC_FLOAT_RANGE(CosfOp); }
+TEST_F(LlvmLibcVecCosfTest, InFloatRange) {
+ TEST_MATHVEC_FLOAT_RANGE_TOL(CosfOp, MATHVEC_TOL);
+}
diff --git a/libc/test/src/mathvec/exhaustive/cosf_test.cpp b/libc/test/src/mathvec/exhaustive/cosf_test.cpp
index 13ff8fee37d40..caa5f034fea2a 100644
--- a/libc/test/src/mathvec/exhaustive/cosf_test.cpp
+++ b/libc/test/src/mathvec/exhaustive/cosf_test.cpp
@@ -13,12 +13,20 @@
#include "exhaustive_test.h"
#include "src/__support/CPP/simd.h"
-#include "src/math/cosf.h"
+#include "src/__support/macros/optimization.h"
+#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+#define MATHVEC_TOL 1
+#else
+#define MATHVEC_TOL 0
+#endif
+// Keep the scalar reference correctly rounded in every build configuration.
+#undef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+#include "src/__support/math/cosf_double_eval.h"
#include "src/mathvec/cosf.h"
-using LlvmLibcCosfExhaustiveTest =
- LlvmLibcUnaryOpExhaustiveMathvecTest<float, LIBC_NAMESPACE::cosf,
- LIBC_NAMESPACE::cosf>;
+using LlvmLibcCosfExhaustiveTest = LlvmLibcUnaryOpExhaustiveMathvecTest<
+ float, LIBC_NAMESPACE::math::double_eval::cosf, LIBC_NAMESPACE::cosf,
+ MATHVEC_TOL>;
// Tests all possible 32-bit input patterns
TEST_F(LlvmLibcCosfExhaustiveTest, EntireRange) { test_full_range_RN(); }
diff --git a/libc/test/src/mathvec/exhaustive/exhaustive_test.h b/libc/test/src/mathvec/exhaustive/exhaustive_test.h
index 5256f69fd5a36..b94cbfa09a58a 100644
--- a/libc/test/src/mathvec/exhaustive/exhaustive_test.h
+++ b/libc/test/src/mathvec/exhaustive/exhaustive_test.h
@@ -13,7 +13,7 @@
#include "src/__support/CPP/limits.h"
#include "src/__support/CPP/simd.h"
-#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/SIMDMatcher.h"
#include <atomic>
#include <iostream>
@@ -43,7 +43,7 @@ using VectorUnaryOp =
template <typename OutType, typename InType,
ScalarUnaryOp<OutType, InType> ScalarFunc,
- VectorUnaryOp<OutType, InType> VectorFunc>
+ VectorUnaryOp<OutType, InType> VectorFunc, uint64_t TOL = 0>
struct UnaryOpChecker : public virtual LIBC_NAMESPACE::testing::Test {
using FloatType = InType;
using FPBits = LIBC_NAMESPACE::fputil::FPBits<FloatType>;
@@ -66,10 +66,16 @@ struct UnaryOpChecker : public virtual LIBC_NAMESPACE::testing::Test {
LIBC_NAMESPACE::cpp::simd<OutType> vec_result = VectorFunc(vec_x);
OutType vec_res = vec_result[0];
OutType scalar_result = ScalarFunc(x);
- bool correct = TEST_FP_EQ(scalar_result, vec_res);
+ bool correct = TOL == 0 ? TEST_FP_EQ(scalar_result, vec_res)
+ : LIBC_NAMESPACE::testing::within_ulp_tolerance(
+ scalar_result, vec_res, TOL);
if (!correct) {
- EXPECT_FP_EQ(scalar_result, vec_res);
+ if constexpr (TOL == 0)
+ EXPECT_FP_EQ(scalar_result, vec_res);
+ else
+ EXPECT_TRUE(LIBC_NAMESPACE::testing::within_ulp_tolerance(
+ scalar_result, vec_res, TOL));
failed++;
}
} while (bits++ < stop);
@@ -222,6 +228,6 @@ struct LlvmLibcExhaustiveMathvecTest
};
template <typename FloatType, ScalarUnaryOp<FloatType> ScalarFunc,
- VectorUnaryOp<FloatType> VectorFunc>
+ VectorUnaryOp<FloatType> VectorFunc, uint64_t TOL = 0>
using LlvmLibcUnaryOpExhaustiveMathvecTest = LlvmLibcExhaustiveMathvecTest<
- UnaryOpChecker<FloatType, FloatType, ScalarFunc, VectorFunc>>;
+ UnaryOpChecker<FloatType, FloatType, ScalarFunc, VectorFunc, TOL>>;
diff --git a/libc/test/src/mathvec/exhaustive/sinf_test.cpp b/libc/test/src/mathvec/exhaustive/sinf_test.cpp
index e72fcea4bc0b7..d4762394f73f5 100644
--- a/libc/test/src/mathvec/exhaustive/sinf_test.cpp
+++ b/libc/test/src/mathvec/exhaustive/sinf_test.cpp
@@ -13,12 +13,14 @@
#include "exhaustive_test.h"
#include "src/__support/CPP/simd.h"
-#include "src/math/sinf.h"
+#include "src/__support/macros/optimization.h"
+// Keep the scalar reference correctly rounded in every build configuration.
+#undef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+#include "src/__support/math/sinf_double_eval.h"
#include "src/mathvec/sinf.h"
-using LlvmLibcSinfExhaustiveTest =
- LlvmLibcUnaryOpExhaustiveMathvecTest<float, LIBC_NAMESPACE::sinf,
- LIBC_NAMESPACE::sinf>;
+using LlvmLibcSinfExhaustiveTest = LlvmLibcUnaryOpExhaustiveMathvecTest<
+ float, LIBC_NAMESPACE::math::double_eval::sinf, LIBC_NAMESPACE::sinf>;
// Tests all possible 32-bit input patterns
TEST_F(LlvmLibcSinfExhaustiveTest, EntireRange) { test_full_range_RN(); }
diff --git a/libc/test/src/mathvec/sinf_test.cpp b/libc/test/src/mathvec/sinf_test.cpp
index 8aacf20e3c5a4..7d29fa0942f5e 100644
--- a/libc/test/src/mathvec/sinf_test.cpp
+++ b/libc/test/src/mathvec/sinf_test.cpp
@@ -14,7 +14,10 @@
#include "hdr/math_macros.h"
#include "src/__support/CPP/simd.h"
#include "src/__support/FPUtil/FPBits.h"
-#include "src/math/sinf.h"
+#include "src/__support/macros/optimization.h"
+// Keep the scalar reference correctly rounded in every build configuration.
+#undef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+#include "src/__support/math/sinf_double_eval.h"
#include "src/mathvec/sinf.h"
#include "test/UnitTest/SIMDMatcher.h"
#include "test/UnitTest/Test.h"
@@ -25,9 +28,8 @@
using LlvmLibcVecSinfTest = LIBC_NAMESPACE::testing::FPTest<float>;
-using SinfOp =
- LIBC_NAMESPACE::testing::mathvec::UnaryOp<float, LIBC_NAMESPACE::sinf,
- LIBC_NAMESPACE::sinf>;
+using SinfOp = LIBC_NAMESPACE::testing::mathvec::UnaryOp<
+ float, LIBC_NAMESPACE::math::double_eval::sinf, LIBC_NAMESPACE::sinf>;
using LIBC_NAMESPACE::cpp::splat;
using LIBC_NAMESPACE::testing::SDCOMP26094_VALUES;
using LIBC_NAMESPACE::testing::mathvec::wrap_ref;
More information about the libc-commits
mailing list