[libc] [llvm] Add vector-based strlen implementation for x86_64 and aarch64 (PR #152389)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 6 14:22:27 PDT 2025
github-actions[bot] wrote:
<!--LLVM CODE FORMAT COMMENT: {clang-format}-->
:warning: C/C++ code formatter, clang-format found issues in your code. :warning:
<details>
<summary>
You can test this locally with the following command:
</summary>
``````````bash
git-clang-format --diff HEAD~1 HEAD --extensions h -- libc/src/string/inline_strlen.h libc/src/string/memory_utils/aarch64/inline_strlen.h libc/src/string/memory_utils/x86_64/inline_strlen.h libc/src/string/string_utils.h
``````````
</details>
<details>
<summary>
View the diff from clang-format here.
</summary>
``````````diff
diff --git a/libc/src/string/memory_utils/aarch64/inline_strlen.h b/libc/src/string/memory_utils/aarch64/inline_strlen.h
index 2b9f226d1..2bd1c981d 100644
--- a/libc/src/string/memory_utils/aarch64/inline_strlen.h
+++ b/libc/src/string/memory_utils/aarch64/inline_strlen.h
@@ -13,7 +13,7 @@
namespace LIBC_NAMESPACE_DECL {
-size_t string_length_neon(const char* src) {
+size_t string_length_neon(const char *src) {
using Vector __attribute__((may_alias)) = uint8x8_t;
uintptr_t misalign_bytes = reinterpret_case<uintptr_t>(src) % sizeof(Vector);
Vector *block_ptr = reinterpret_cast<Vector *>(src - misalign_bytes);
@@ -23,7 +23,8 @@ size_t string_length_neon(const char* src) {
uint64x1_t cmp_mask = vreinterpret_u64_s8(vcmp);
uint64_t cmp = vget_lane_u64(cmp_mask, 0);
cmp = cmp >> (misalign_bytes << 3);
- if (cmp) return __builtin_ctzl(cmp) >> 3;
+ if (cmp)
+ return __builtin_ctzl(cmp) >> 3;
++block_ptr;
}
while (true) {
@@ -46,5 +47,4 @@ template <typename T>
} // namespace LIBC_NAMESPACE_DECL
-
#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_STRLEN_H
diff --git a/libc/src/string/memory_utils/x86_64/inline_strlen.h b/libc/src/string/memory_utils/x86_64/inline_strlen.h
index ffdf8938d..ec0f67982 100644
--- a/libc/src/string/memory_utils/x86_64/inline_strlen.h
+++ b/libc/src/string/memory_utils/x86_64/inline_strlen.h
@@ -8,7 +8,7 @@
#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_STRLEN_H
#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_STRLEN_H
-#include "src/string/memory_utils/op_x86.h" // K_AVX
+#include "src/string/memory_utils/op_x86.h" // K_AVX
#include <stddef.h> // size_t
#include <x86intrin.h>
@@ -19,28 +19,27 @@ namespace LIBC_NAMESPACE_DECL {
using Vector __attribute__((may_alias)) = __m128i;
Vector z = _mm_setzero_si128();
uintptr_t misalign_bytes = reinterpret_cast<uintptr_t>(src) % sizeof(Vector);
- const Vector *block_ptr = reinterpret_cast<const Vector *>(src - misalign_bytes);
- if (misalign_bytes)
- {
- Vector v = _mm_load_si128 (block_ptr);
- Vector vcmp = _mm_cmpeq_epi8 (z, v);
- // shift away results in irrelevant bytes.
- int cmp = _mm_movemask_epi8 (vcmp) >> misalign_bytes;
- if (cmp)
- return __builtin_ctz (cmp);
- block_ptr++;
- }
- while (true)
- {
- Vector v = _mm_load_si128 (block_ptr);
- Vector vcmp = _mm_cmpeq_epi8 (z, v);
- int cmp = _mm_movemask_epi8 (vcmp);
- if (cmp)
- return static_cast<size_t>(reinterpret_cast<uintptr_t>(block_ptr) -
- reinterpret_cast<uintptr_t>(src) +
- __builtin_ctz(cmp));
- block_ptr++;
- }
+ const Vector *block_ptr =
+ reinterpret_cast<const Vector *>(src - misalign_bytes);
+ if (misalign_bytes) {
+ Vector v = _mm_load_si128(block_ptr);
+ Vector vcmp = _mm_cmpeq_epi8(z, v);
+ // shift away results in irrelevant bytes.
+ int cmp = _mm_movemask_epi8(vcmp) >> misalign_bytes;
+ if (cmp)
+ return __builtin_ctz(cmp);
+ block_ptr++;
+ }
+ while (true) {
+ Vector v = _mm_load_si128(block_ptr);
+ Vector vcmp = _mm_cmpeq_epi8(z, v);
+ int cmp = _mm_movemask_epi8(vcmp);
+ if (cmp)
+ return static_cast<size_t>(reinterpret_cast<uintptr_t>(block_ptr) -
+ reinterpret_cast<uintptr_t>(src) +
+ __builtin_ctz(cmp));
+ block_ptr++;
+ }
}
#endif
@@ -49,37 +48,37 @@ namespace LIBC_NAMESPACE_DECL {
using Vector __attribute__((may_alias)) = __mm256i;
Vector z = _mm256_setzero_si256();
uintptr_t misalign_bytes = reinterpret_cast<uintptr_t>(src) % sizeof(Vector);
- const Vector *block_ptr = reinterpret_cast<const Vector *>(src - misalign_bytes);
- if (misalign_bytes)
- {
- Vector v = _mm256_load_si256 (block_ptr);
- Vector vcmp = _mm256_cmpeq_epi8 (z, v);
- // shift away results in irrelevant bytes.
- int cmp = _mm256_movemask_epi8 (vcmp) >> misalign_bytes;
- if (cmp)
- return __builtin_ctz(cmp);
- block_ptr++;
+ const Vector *block_ptr =
+ reinterpret_cast<const Vector *>(src - misalign_bytes);
+ if (misalign_bytes) {
+ Vector v = _mm256_load_si256(block_ptr);
+ Vector vcmp = _mm256_cmpeq_epi8(z, v);
+ // shift away results in irrelevant bytes.
+ int cmp = _mm256_movemask_epi8(vcmp) >> misalign_bytes;
+ if (cmp)
+ return __builtin_ctz(cmp);
+ block_ptr++;
+ }
+ while (true) {
+ Vector v = _mm256_load_si256(block_ptr);
+ Vector vcmp = _mm256_cmpeq_epi8(z, v);
+ int cmp = _mm256_movemask_epi8(vcmp);
+ if (cmp)
+ return static_cast<size_t>(reinterpret_cast<uintptr_t>(block_ptr) -
+ reinterpret_cast<uintptr_t>(src) +
+ __builtin_ctz(cmp));
+ block_ptr++;
}
- while (true)
- {
- Vector v = _mm256_load_si256 (block_ptr);
- Vector vcmp = _mm256_cmpeq_epi8 (z, v);
- int cmp = _mm256_movemask_epi8 (vcmp);
- if (cmp)
- return static_cast<size_t>(reinterpret_cast<uintptr_t>(block_ptr) -
- reinterpret_cast<uintptr_t>(src) +
- __builtin_ctz(cmp));
- block_ptr++;
- }
}
-#endif // __AVX__
+#endif // __AVX__
#if defined(__AVX512F__)
[[maybe_unused]] LIBC_INLINE size_t string_length_avx512(const char *src) {
using Vector __attribute__((may_alias)) = __mm512i;
Vector z = _mm512_setzero_si512();
uintptr_t misalign_bytes = reinterpret_cast<uintptr_t>(src) % sizeof(Vector);
- const Vector *block_ptr = reinterpret_cast<const Vector *>(src - misalign_bytes);
+ const Vector *block_ptr =
+ reinterpret_cast<const Vector *>(src - misalign_bytes);
if (misalign_bytes) {
Vector v = _mm512_load_si512(block_ptr);
__mmask64 cmp = _mm512_cmp_epu8_mask(z, v, _MM_CMPINT_EQ) >> misalign_bytes;
@@ -87,8 +86,7 @@ namespace LIBC_NAMESPACE_DECL {
return __builtin_ctzl(cmp);
block_ptr++;
}
- while (true)
- {
+ while (true) {
Vector v = _mm512_load_si512(block_ptr);
__mmask64 cmp = _mm512_cmp_epu8_mask(z, v, _MM_CMPINT_EQ);
if (cmp)
@@ -96,12 +94,11 @@ namespace LIBC_NAMESPACE_DECL {
reinterpret_cast<uintptr_t>(src) +
__builtin_ctz(cmp));
block_ptr++;
- }
+ }
}
-#endif // __AVX512F__
+#endif // __AVX512F__
-template<typename T> LIBC_INLINE
-size_t string_length_x86_64(const char *src) {
+template <typename T> LIBC_INLINE size_t string_length_x86_64(const char *src) {
#if defined(__AVX512F__)
return string_length_avx512(src);
#endif
diff --git a/libc/src/string/string_utils.h b/libc/src/string/string_utils.h
index 8312ef895..c9baf5a21 100644
--- a/libc/src/string/string_utils.h
+++ b/libc/src/string/string_utils.h
@@ -22,7 +22,6 @@
#include "src/__support/macros/config.h"
#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
-
#if defined(LIBC_COPT_STRING_UNSAFE_WIDE_READ)
#if defined(LIBC_TARGET_ARCH_IS_X86)
#include "src/string/memory_utils/x86_64/inline_strlen.h"
``````````
</details>
https://github.com/llvm/llvm-project/pull/152389
More information about the llvm-commits
mailing list