[libcxx-commits] [libcxx] [libc++] Speed-up reduction by up to 4X by breaking the dependency chain (PR #234887)
Michael G. Kazakov via libcxx-commits
libcxx-commits at lists.llvm.org
Fri Oct 23 02:43:49 PDT 2026
https://github.com/mikekazakov created https://github.com/llvm/llvm-project/pull/234887
None
>From 2656284cee5fb48867497e10f053b59d95722472 Mon Sep 17 00:00:00 2001
From: Michael Kazakov <mike.kazakov at gmail.com>
Date: Mon, 19 Oct 2026 09:57:03 +0100
Subject: [PATCH 1/3] Add benchmarks for reduce() and transform_reduce()
---
.../test/benchmarks/numeric/reduce.bench.cpp | 72 ++++++++++++
.../numeric/transform_reduce.bench.cpp | 109 ++++++++++++++++++
2 files changed, 181 insertions(+)
create mode 100644 libcxx/test/benchmarks/numeric/reduce.bench.cpp
create mode 100644 libcxx/test/benchmarks/numeric/transform_reduce.bench.cpp
diff --git a/libcxx/test/benchmarks/numeric/reduce.bench.cpp b/libcxx/test/benchmarks/numeric/reduce.bench.cpp
new file mode 100644
index 00000000000000..c68aa794d8f5e4
--- /dev/null
+++ b/libcxx/test/benchmarks/numeric/reduce.bench.cpp
@@ -0,0 +1,72 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+
+#include <algorithm>
+#include <cstddef>
+#include <iterator>
+#include <list>
+#include <numeric>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+#include <benchmark/benchmark.h>
+#include "../GenerateInput.h"
+#include "test_macros.h"
+
+// Keep values small so that signed integer arithmetic does not overflow even for the largest inputs
+template <class T>
+T random_value() {
+ if constexpr (std::is_integral_v<T>)
+ return getRandomInteger<T>(-100, 100);
+ else
+ return getRandomFloat<T>(-100, 100);
+}
+
+int main(int argc, char** argv) {
+ // std::reduce(first, last, init, reduce)
+ {
+ auto bm = []<class Container>(std::string name) {
+ benchmark::RegisterBenchmark(
+ name,
+ [](auto& st) TEST_ALIGN_BENCHMARK {
+ std::size_t const size = st.range(0);
+ using ValueType = typename Container::value_type;
+
+ Container c;
+ std::generate_n(std::back_inserter(c), size, [] { return random_value<ValueType>(); });
+ ValueType init = random_value<ValueType>();
+
+ auto reduce = [](auto x, auto y) { return x + y; };
+
+ for ([[maybe_unused]] auto _ : st) {
+ benchmark::ClobberMemory();
+ auto result = std::reduce(c.begin(), c.end(), init, reduce);
+ benchmark::DoNotOptimize(result);
+ }
+ })
+ ->Arg(32)
+ ->Arg(50) // non power-of-two
+ ->Arg(1024)
+ ->Arg(8192);
+ };
+ bm.operator()<std::vector<int>>("std::reduce(vector<int>)");
+ bm.operator()<std::vector<float>>("std::reduce(vector<float>)");
+ bm.operator()<std::vector<double>>("std::reduce(vector<double>)");
+ bm.operator()<std::list<int>>("std::reduce(list<int>)");
+ bm.operator()<std::list<float>>("std::reduce(list<float>)");
+ bm.operator()<std::list<double>>("std::reduce(list<double>)");
+ }
+
+ benchmark::Initialize(&argc, argv);
+ benchmark::RunSpecifiedBenchmarks();
+ benchmark::Shutdown();
+ return 0;
+}
diff --git a/libcxx/test/benchmarks/numeric/transform_reduce.bench.cpp b/libcxx/test/benchmarks/numeric/transform_reduce.bench.cpp
new file mode 100644
index 00000000000000..a0cde2f47aa980
--- /dev/null
+++ b/libcxx/test/benchmarks/numeric/transform_reduce.bench.cpp
@@ -0,0 +1,109 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+
+#include <algorithm>
+#include <cstddef>
+#include <iterator>
+#include <list>
+#include <numeric>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+#include <benchmark/benchmark.h>
+#include "../GenerateInput.h"
+#include "test_macros.h"
+
+// Keep values small so that signed integer arithmetic does not overflow even for the largest inputs
+template <class T>
+T random_value() {
+ if constexpr (std::is_integral_v<T>)
+ return getRandomInteger<T>(-100, 100);
+ else
+ return getRandomFloat<T>(-100, 100);
+}
+
+int main(int argc, char** argv) {
+ // std::transform_reduce(first, last, init, reduce, transform)
+ {
+ auto bm = []<class Container>(std::string name) {
+ benchmark::RegisterBenchmark(
+ name,
+ [](auto& st) TEST_ALIGN_BENCHMARK {
+ std::size_t const size = st.range(0);
+ using ValueType = typename Container::value_type;
+
+ Container c;
+ std::generate_n(std::back_inserter(c), size, [] { return random_value<ValueType>(); });
+ ValueType init = random_value<ValueType>();
+
+ auto transform = [](auto x) { return x * 3; };
+ auto reduce = [](auto x, auto y) { return x + y; };
+
+ for ([[maybe_unused]] auto _ : st) {
+ benchmark::ClobberMemory();
+ auto result = std::transform_reduce(c.begin(), c.end(), init, reduce, transform);
+ benchmark::DoNotOptimize(result);
+ }
+ })
+ ->Arg(32)
+ ->Arg(50) // non power-of-two
+ ->Arg(1024)
+ ->Arg(8192);
+ };
+ bm.operator()<std::vector<int>>("std::transform_reduce(vector<int>)");
+ bm.operator()<std::vector<float>>("std::transform_reduce(vector<float>)");
+ bm.operator()<std::vector<double>>("std::transform_reduce(vector<double>)");
+ bm.operator()<std::list<int>>("std::transform_reduce(list<int>)");
+ bm.operator()<std::list<float>>("std::transform_reduce(list<float>)");
+ bm.operator()<std::list<double>>("std::transform_reduce(list<double>)");
+ }
+
+ // std::transform_reduce(first1, last1, first2, init, reduce, transform)
+ {
+ auto bm = []<class Container>(std::string name) {
+ benchmark::RegisterBenchmark(
+ name,
+ [](auto& st) TEST_ALIGN_BENCHMARK {
+ std::size_t const size = st.range(0);
+ using ValueType = typename Container::value_type;
+
+ Container c1, c2;
+ std::generate_n(std::back_inserter(c1), size, [] { return random_value<ValueType>(); });
+ std::generate_n(std::back_inserter(c2), size, [] { return random_value<ValueType>(); });
+ ValueType init = random_value<ValueType>();
+
+ auto transform = [](auto x, auto y) { return x - y; };
+ auto reduce = [](auto x, auto y) { return x + y; };
+
+ for ([[maybe_unused]] auto _ : st) {
+ benchmark::ClobberMemory();
+ auto result = std::transform_reduce(c1.begin(), c1.end(), c2.begin(), init, reduce, transform);
+ benchmark::DoNotOptimize(result);
+ }
+ })
+ ->Arg(32)
+ ->Arg(50) // non power-of-two
+ ->Arg(1024)
+ ->Arg(8192);
+ };
+ bm.operator()<std::vector<int>>("std::transform_reduce(vector<int>, vector<int>)");
+ bm.operator()<std::vector<float>>("std::transform_reduce(vector<float>, vector<float>)");
+ bm.operator()<std::vector<double>>("std::transform_reduce(vector<double>, vector<double>)");
+ bm.operator()<std::list<int>>("std::transform_reduce(list<int>, list<int>)");
+ bm.operator()<std::list<float>>("std::transform_reduce(list<float>, list<float>)");
+ bm.operator()<std::list<double>>("std::transform_reduce(list<double>, list<double>)");
+ }
+
+ benchmark::Initialize(&argc, argv);
+ benchmark::RunSpecifiedBenchmarks();
+ benchmark::Shutdown();
+ return 0;
+}
>From 1267c85d4e1d5329b3546038bb8c673f96db14f3 Mon Sep 17 00:00:00 2001
From: Michael Kazakov <mike.kazakov at gmail.com>
Date: Thu, 22 Oct 2026 11:10:14 +0100
Subject: [PATCH 2/3] Make the reductions heavier to keep the overall results
less flaky
---
.../test/benchmarks/numeric/reduce.bench.cpp | 22 +++++++++----
.../numeric/transform_reduce.bench.cpp | 32 +++++++++++++++----
2 files changed, 40 insertions(+), 14 deletions(-)
diff --git a/libcxx/test/benchmarks/numeric/reduce.bench.cpp b/libcxx/test/benchmarks/numeric/reduce.bench.cpp
index c68aa794d8f5e4..ece7578d18ff32 100644
--- a/libcxx/test/benchmarks/numeric/reduce.bench.cpp
+++ b/libcxx/test/benchmarks/numeric/reduce.bench.cpp
@@ -21,13 +21,13 @@
#include "../GenerateInput.h"
#include "test_macros.h"
-// Keep values small so that signed integer arithmetic does not overflow even for the largest inputs
+// Keep values small so that signed integer multiplication doesn't overflow
template <class T>
T random_value() {
if constexpr (std::is_integral_v<T>)
- return getRandomInteger<T>(-100, 100);
+ return getRandomInteger<T>(1, 1000);
else
- return getRandomFloat<T>(-100, 100);
+ return getRandomFloat<T>(1, 1000);
}
int main(int argc, char** argv) {
@@ -42,11 +42,19 @@ int main(int argc, char** argv) {
Container c;
std::generate_n(std::back_inserter(c), size, [] { return random_value<ValueType>(); });
- ValueType init = random_value<ValueType>();
+ ValueType init = static_cast<ValueType>(1);
- auto reduce = [](auto x, auto y) { return x + y; };
-
- for ([[maybe_unused]] auto _ : st) {
+ auto reduce = [](auto x, auto y) {
+ if constexpr (std::is_floating_point_v<decltype(x)>) {
+ // Compilers aren't allowed to optimize away the FP divisions and multiplications without fast-math.
+ return x / 3. * 3. / 5. * 5. + y / 7. * 7. / 9. * 9.;
+ } else {
+ // p=46337 is the largest prime with '(p-1)^2 <= INT_MAX', so the chain of reductions never overflows.
+ constexpr decltype(x) p = 46337;
+ return x * y % p;
+ }
+ };
+ for (auto _ : st) {
benchmark::ClobberMemory();
auto result = std::reduce(c.begin(), c.end(), init, reduce);
benchmark::DoNotOptimize(result);
diff --git a/libcxx/test/benchmarks/numeric/transform_reduce.bench.cpp b/libcxx/test/benchmarks/numeric/transform_reduce.bench.cpp
index a0cde2f47aa980..192308f1a9031d 100644
--- a/libcxx/test/benchmarks/numeric/transform_reduce.bench.cpp
+++ b/libcxx/test/benchmarks/numeric/transform_reduce.bench.cpp
@@ -21,13 +21,13 @@
#include "../GenerateInput.h"
#include "test_macros.h"
-// Keep values small so that signed integer arithmetic does not overflow even for the largest inputs
+// Keep values small so that signed integer multiplication doesn't overflow
template <class T>
T random_value() {
if constexpr (std::is_integral_v<T>)
- return getRandomInteger<T>(-100, 100);
+ return getRandomInteger<T>(1, 1000);
else
- return getRandomFloat<T>(-100, 100);
+ return getRandomFloat<T>(1, 1000);
}
int main(int argc, char** argv) {
@@ -42,10 +42,19 @@ int main(int argc, char** argv) {
Container c;
std::generate_n(std::back_inserter(c), size, [] { return random_value<ValueType>(); });
- ValueType init = random_value<ValueType>();
+ ValueType init = static_cast<ValueType>(1);
auto transform = [](auto x) { return x * 3; };
- auto reduce = [](auto x, auto y) { return x + y; };
+ auto reduce = [](auto x, auto y) {
+ if constexpr (std::is_floating_point_v<decltype(x)>) {
+ // Compilers aren't allowed to optimize away the FP divisions and multiplications without fast-math.
+ return x / 3. * 3. / 5. * 5. + y / 7. * 7. / 9. * 9.;
+ } else {
+ // p=46337 is the largest prime with '(p-1)^2 <= INT_MAX', so the chain of reductions never overflows.
+ constexpr decltype(x) p = 46337;
+ return x * y % p;
+ }
+ };
for ([[maybe_unused]] auto _ : st) {
benchmark::ClobberMemory();
@@ -78,10 +87,19 @@ int main(int argc, char** argv) {
Container c1, c2;
std::generate_n(std::back_inserter(c1), size, [] { return random_value<ValueType>(); });
std::generate_n(std::back_inserter(c2), size, [] { return random_value<ValueType>(); });
- ValueType init = random_value<ValueType>();
+ ValueType init = static_cast<ValueType>(1);
auto transform = [](auto x, auto y) { return x - y; };
- auto reduce = [](auto x, auto y) { return x + y; };
+ auto reduce = [](auto x, auto y) {
+ if constexpr (std::is_floating_point_v<decltype(x)>) {
+ // Compilers aren't allowed to optimize away the FP divisions and multiplications without fast-math.
+ return x / 3. * 3. / 5. * 5. + y / 7. * 7. / 9. * 9.;
+ } else {
+ // p=46337 is the largest prime with '(p-1)^2 <= INT_MAX', so the chain of reductions never overflows.
+ constexpr decltype(x) p = 46337;
+ return x * y % p;
+ }
+ };
for ([[maybe_unused]] auto _ : st) {
benchmark::ClobberMemory();
>From e4d9050cb95b0820e25543d0c374cb3e10fa9ac4 Mon Sep 17 00:00:00 2001
From: Michael Kazakov <mike.kazakov at gmail.com>
Date: Fri, 23 Oct 2026 10:41:15 +0100
Subject: [PATCH 3/3] Use 4-way tree reduction
---
libcxx/include/__numeric/reduce.h | 8 ++++++--
libcxx/include/__numeric/transform_reduce.h | 17 +++++++++++++++--
2 files changed, 21 insertions(+), 4 deletions(-)
diff --git a/libcxx/include/__numeric/reduce.h b/libcxx/include/__numeric/reduce.h
index 853ca6fb61860e..fe245e90978016 100644
--- a/libcxx/include/__numeric/reduce.h
+++ b/libcxx/include/__numeric/reduce.h
@@ -27,8 +27,12 @@ _LIBCPP_PUSH_MACROS
_LIBCPP_BEGIN_NAMESPACE_STD
template <class _InputIterator, class _Tp, class _BinaryOp>
-[[nodiscard]] _LIBCPP_HIDE_FROM_ABI
-_LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOp __b) {
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp
+reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOp __b) {
+ if constexpr (__has_random_access_iterator_category<_InputIterator>::value) {
+ for (; __last - __first >= 4; __first += 4)
+ __init = __b(std::move(__init), __b(__b(__first[0], __first[1]), __b(__first[2], __first[3])));
+ }
for (; __first != __last; ++__first)
__init = __b(std::move(__init), *__first);
return __init;
diff --git a/libcxx/include/__numeric/transform_reduce.h b/libcxx/include/__numeric/transform_reduce.h
index 53ed6aaf32d1ec..66428b328ae5f9 100644
--- a/libcxx/include/__numeric/transform_reduce.h
+++ b/libcxx/include/__numeric/transform_reduce.h
@@ -12,6 +12,7 @@
#include <__config>
#include <__functional/operations.h>
+#include <__iterator/iterator_traits.h>
#include <__utility/move.h>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -26,8 +27,13 @@ _LIBCPP_PUSH_MACROS
_LIBCPP_BEGIN_NAMESPACE_STD
template <class _InputIterator, class _Tp, class _BinaryOp, class _UnaryOp>
-[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
-_Tp transform_reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOp __b, _UnaryOp __u) {
+[[nodiscard]] _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp
+transform_reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOp __b, _UnaryOp __u) {
+ if constexpr (__has_random_access_iterator_category<_InputIterator>::value) {
+ for (; __last - __first >= 4; __first += 4)
+ __init =
+ __b(std::move(__init), __b(__b(__u(__first[0]), __u(__first[1])), __b(__u(__first[2]), __u(__first[3]))));
+ }
for (; __first != __last; ++__first)
__init = __b(std::move(__init), __u(*__first));
return __init;
@@ -41,6 +47,13 @@ template <class _InputIterator1, class _InputIterator2, class _Tp, class _Binary
_Tp __init,
_BinaryOp1 __b1,
_BinaryOp2 __b2) {
+ if constexpr (__has_random_access_iterator_category<_InputIterator1>::value &&
+ __has_random_access_iterator_category<_InputIterator2>::value) {
+ for (; __last1 - __first1 >= 4; __first1 += 4, (void)(__first2 += 4))
+ __init = __b1(std::move(__init),
+ __b1(__b1(__b2(__first1[0], __first2[0]), __b2(__first1[1], __first2[1])),
+ __b1(__b2(__first1[2], __first2[2]), __b2(__first1[3], __first2[3]))));
+ }
for (; __first1 != __last1; ++__first1, (void)++__first2)
__init = __b1(std::move(__init), __b2(*__first1, *__first2));
return __init;
More information about the libcxx-commits
mailing list