[libcxx-commits] [libcxx] 1c06165 - [libc++] Make std::align an inline function (#167472)
via libcxx-commits
libcxx-commits at lists.llvm.org
Thu Dec 18 01:55:33 PST 2025
Author: saipubw
Date: 2025-12-18T17:55:28+08:00
New Revision: 1c06165c9b203a9543bc6c368e680d38158b44a0
URL: https://github.com/llvm/llvm-project/commit/1c06165c9b203a9543bc6c368e680d38158b44a0
DIFF: https://github.com/llvm/llvm-project/commit/1c06165c9b203a9543bc6c368e680d38158b44a0.diff
LOG: [libc++] Make std::align an inline function (#167472)
`std::align` is heavily used in memory allocators. When we attempted to
switch from libstdc++ to libc++, we observed a **50%** performance
regression in a database query bench: the issue is that `std::align` in
libc++ is not an inline function, which prevents the compiler from
performing inlining optimizations.
make `std::align` an inline function will run about 2x faster. See
[benchmark
result](https://quick-bench.com/q/wPTnt9JCGn2S-3bu5gY9YrEf6KU).
---------
Co-authored-by: Louis Dionne <ldionne.2 at gmail.com>
Added:
libcxx/test/benchmarks/memory/align.bench.cpp
Modified:
libcxx/docs/ABIGuarantees.rst
libcxx/docs/ReleaseNotes/22.rst
libcxx/include/__configuration/abi.h
libcxx/include/__memory/align.h
libcxx/src/memory.cpp
Removed:
################################################################################
diff --git a/libcxx/docs/ABIGuarantees.rst b/libcxx/docs/ABIGuarantees.rst
index e680f5429880e..4ca0451e1800c 100644
--- a/libcxx/docs/ABIGuarantees.rst
+++ b/libcxx/docs/ABIGuarantees.rst
@@ -114,6 +114,11 @@ hand, backwards compatibility is generally guaranteed.
There are multiple ABI flags that change the symbols exported from the built library:
+``_LIBCPP_ABI_DO_NOT_EXPORT_ALIGN``
+-------------------------------------------------
+This removes ``std::align()`` from the built library. In the past, ``std::align()`` was defined in the built library,
+but nowadays it is an inline function defined in the headers for performance reasons.
+
``_LIBCPP_ABI_DO_NOT_EXPORT_BASIC_STRING_COMMON``
-------------------------------------------------
This removes ``__basic_string_common<true>::__throw_length_error()`` and
diff --git a/libcxx/docs/ReleaseNotes/22.rst b/libcxx/docs/ReleaseNotes/22.rst
index 57f63db573300..603551cb0f603 100644
--- a/libcxx/docs/ReleaseNotes/22.rst
+++ b/libcxx/docs/ReleaseNotes/22.rst
@@ -88,8 +88,12 @@ Improvements and New Features
- ``std::for_each`` and ``ranges::for_each`` have been optimized to iterate more efficiently over the associative
containers, resulting in performance improvements of up to 2x.
+
- The ``num_get::do_get`` integral overloads have been optimized, resulting in a performance improvement of up to 2.8x.
+- The performance of ``std::align`` has been improved by making it an inline function, which allows the compiler to
+ better optimize calls to it.
+
Deprecations and Removals
-------------------------
@@ -121,6 +125,9 @@ ABI Affecting Changes
potentially inheriting from the types they wrap. At this point in time we are not aware of any ABI changes caused by
this.
+- ``std::align`` is now implemented as an inline function and its definition is removed from the
+ libc++ built library in ABI v2, or when the ``_LIBCPP_ABI_DO_NOT_EXPORT_ALIGN`` ABI configuration is enabled.
+
- ``ranges::iota_view`` is now aware of ``__int128``. This causes ``iota_view::
diff erence_type`` to change from
``long long`` to ``__int128`` in some cases.
diff --git a/libcxx/include/__configuration/abi.h b/libcxx/include/__configuration/abi.h
index f30d0a5445b19..cced9c39749b7 100644
--- a/libcxx/include/__configuration/abi.h
+++ b/libcxx/include/__configuration/abi.h
@@ -64,6 +64,7 @@
// These flags are documented in ABIGuarantees.rst
# define _LIBCPP_ABI_ALTERNATE_STRING_LAYOUT
# define _LIBCPP_ABI_ATOMIC_WAIT_NATIVE_BY_SIZE
+# define _LIBCPP_ABI_DO_NOT_EXPORT_ALIGN
# define _LIBCPP_ABI_DO_NOT_EXPORT_BASIC_STRING_COMMON
# define _LIBCPP_ABI_DO_NOT_EXPORT_VECTOR_BASE_COMMON
# define _LIBCPP_ABI_DO_NOT_EXPORT_TO_CHARS_BASE_10
diff --git a/libcxx/include/__memory/align.h b/libcxx/include/__memory/align.h
index 402eac3380925..47a7a2d274627 100644
--- a/libcxx/include/__memory/align.h
+++ b/libcxx/include/__memory/align.h
@@ -11,6 +11,7 @@
#include <__config>
#include <__cstddef/size_t.h>
+#include <cstdint>
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
# pragma GCC system_header
@@ -18,7 +19,23 @@
_LIBCPP_BEGIN_NAMESPACE_STD
-_LIBCPP_EXPORTED_FROM_ABI void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space);
+inline namespace __align_inline {
+_LIBCPP_HIDE_FROM_ABI inline void* align(size_t __align, size_t __sz, void*& __ptr, size_t& __space) {
+ void* __r = nullptr;
+ if (__sz <= __space) {
+ char* __p1 = static_cast<char*>(__ptr);
+ char* __p2 = reinterpret_cast<char*>(reinterpret_cast<uintptr_t>(__p1 + (__align - 1)) & -__align);
+ size_t __d = static_cast<size_t>(__p2 - __p1);
+ if (__d <= __space - __sz) {
+ __r = __p2;
+ __ptr = __r;
+ __space -= __d;
+ }
+ }
+ return __r;
+}
+
+} // namespace __align_inline
_LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/src/memory.cpp b/libcxx/src/memory.cpp
index 9be40cb9c1285..e7d2dbcbb60c2 100644
--- a/libcxx/src/memory.cpp
+++ b/libcxx/src/memory.cpp
@@ -132,19 +132,13 @@ __sp_mut& __get_sp_mut(const void* p) {
#endif // _LIBCPP_HAS_THREADS
-void* align(size_t alignment, size_t size, void*& ptr, size_t& space) {
- void* r = nullptr;
- if (size <= space) {
- char* p1 = static_cast<char*>(ptr);
- char* p2 = reinterpret_cast<char*>(reinterpret_cast<uintptr_t>(p1 + (alignment - 1)) & -alignment);
- size_t d = static_cast<size_t>(p2 - p1);
- if (d <= space - size) {
- r = p2;
- ptr = r;
- space -= d;
- }
- }
- return r;
+#if !defined(_LIBCPP_ABI_DO_NOT_EXPORT_ALIGN)
+
+_LIBCPP_EXPORTED_FROM_ABI void* align(size_t alignment, size_t size, void*& ptr, size_t& space);
+_LIBCPP_EXPORTED_FROM_ABI void* align(size_t alignment, size_t size, void*& ptr, size_t& space) {
+ return __align_inline::align(alignment, size, ptr, space);
}
+#endif // _LIBCPP_ABI_DO_NOT_EXPORT_ALIGN
+
_LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/test/benchmarks/memory/align.bench.cpp b/libcxx/test/benchmarks/memory/align.bench.cpp
new file mode 100644
index 0000000000000..9fb637c21f9ac
--- /dev/null
+++ b/libcxx/test/benchmarks/memory/align.bench.cpp
@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03
+
+#include <memory>
+#include <iostream>
+
+#include "benchmark/benchmark.h"
+
+struct Input {
+ std::size_t align;
+ std::size_t size;
+ void* ptr;
+ std::size_t buffer_size;
+};
+
+static void BM_align(benchmark::State& state) {
+ char buffer[1024];
+ Input input{};
+ void* ptr = buffer + 123;
+ std::size_t buffer_size = sizeof(buffer) - 123;
+ input.align = state.range();
+ input.size = state.range();
+ for (auto _ : state) {
+ input.ptr = ptr;
+ input.buffer_size = buffer_size;
+ benchmark::DoNotOptimize(input);
+ benchmark::DoNotOptimize(std::align(input.align, input.size, input.ptr, input.buffer_size));
+ }
+}
+BENCHMARK(BM_align)->Range(1, 256);
+
+BENCHMARK_MAIN();
More information about the libcxx-commits
mailing list