[libc-commits] [llvm] [clang-tools-extra] [clang] [libc] [libc] Adding a version of memset with software prefetching (PR #70493)
via libc-commits
libc-commits at lists.llvm.org
Tue Oct 31 12:24:04 PDT 2023
https://github.com/doshimili updated https://github.com/llvm/llvm-project/pull/70493
>From 6c313955185c0d59564f6535b6f1580dca168bea Mon Sep 17 00:00:00 2001
From: Your Name <milidoshi at google.com>
Date: Tue, 24 Oct 2023 21:15:23 +0000
Subject: [PATCH 01/10] Add software prefetching to memset
---
libc/src/string/memory_utils/op_generic.h | 19 +++++++++++++++++++
.../memory_utils/x86_64/inline_memset.h | 12 +++++++++++-
2 files changed, 30 insertions(+), 1 deletion(-)
diff --git a/libc/src/string/memory_utils/op_generic.h b/libc/src/string/memory_utils/op_generic.h
index fd71ca30e24b936..54af7ea10e25e46 100644
--- a/libc/src/string/memory_utils/op_generic.h
+++ b/libc/src/string/memory_utils/op_generic.h
@@ -163,6 +163,25 @@ template <typename T> struct Memset {
} while (offset < count - SIZE);
tail(dst, value, count);
}
+
+ template <size_t prefetch_distance, size_t prefetch_degree>
+ LIBC_INLINE static void loop_and_tail_prefetch(Ptr dst, uint8_t value,
+ size_t count) {
+ Memset<uint512_t>::block(dst, value);
+ Memset<uint256_t>::block(dst + 64, value);
+ size_t offset = 96;
+ while (offset + prefetch_degree + kSize <= count) {
+ for (size_t i = 0; i < prefetch_degree / kCachelineSize; ++i)
+ PrefetchW(dst + offset + prefetch_distance + kCachelineSize * i);
+ for (size_t i = 0; i < prefetch_degree; i += kSize, offset += kSize)
+ block(dst + offset, value);
+ }
+ while (offset + kSize < count) {
+ block(dst + offset, value);
+ offset += kSize;
+ }
+ tail(dst, value, count);
+ }
};
template <typename T, typename... TS> struct MemsetSequence {
diff --git a/libc/src/string/memory_utils/x86_64/inline_memset.h b/libc/src/string/memory_utils/x86_64/inline_memset.h
index 6436594856b0eaf..da463bc0029f9aa 100644
--- a/libc/src/string/memory_utils/x86_64/inline_memset.h
+++ b/libc/src/string/memory_utils/x86_64/inline_memset.h
@@ -17,6 +17,11 @@
namespace LIBC_NAMESPACE {
+static constexpr size_t kCachelineSize = 64;
+
+// prefetch for write
+static inline void PrefetchW(CPtr dst) { __builtin_prefetch(dst, 1, 3); }
+
[[maybe_unused]] LIBC_INLINE static void
inline_memset_x86(Ptr dst, uint8_t value, size_t count) {
#if defined(__AVX512F__)
@@ -53,12 +58,17 @@ inline_memset_x86(Ptr dst, uint8_t value, size_t count) {
return generic::Memset<uint128_t>::head_tail(dst, value, count);
if (count <= 64)
return generic::Memset<uint256_t>::head_tail(dst, value, count);
+ PrefetchW(dst + kCachelineSize);
if (count <= 128)
return generic::Memset<uint512_t>::head_tail(dst, value, count);
+ PrefetchW(dst + kCachelineSize * 2);
// Aligned loop
generic::Memset<uint256_t>::block(dst, value);
align_to_next_boundary<32>(dst, count);
- return generic::Memset<uint256_t>::loop_and_tail(dst, value, count);
+ if (count <= 192) {
+ return Memset<uint256_t>::loop_and_tail(dst, value, count);
+ }
+ return Memset<uint256_t>::loop_and_tail_prefetch<320, 128>(dst, value, count);
}
} // namespace LIBC_NAMESPACE
>From 15cbd0a0c851fa3ac5315e796bb69c1bf791e956 Mon Sep 17 00:00:00 2001
From: Your Name <milidoshi at google.com>
Date: Tue, 24 Oct 2023 21:15:23 +0000
Subject: [PATCH 02/10] Add software prefetching to memset
---
libc/src/string/CMakeLists.txt | 1 +
.../memory_utils/x86_64/inline_memset.h | 32 ++++++++++++++++---
.../llvm-project-overlay/libc/BUILD.bazel | 1 +
3 files changed, 29 insertions(+), 5 deletions(-)
diff --git a/libc/src/string/CMakeLists.txt b/libc/src/string/CMakeLists.txt
index 67675b682081c67..aa69bff7a8cfada 100644
--- a/libc/src/string/CMakeLists.txt
+++ b/libc/src/string/CMakeLists.txt
@@ -656,6 +656,7 @@ if(${LIBC_TARGET_ARCHITECTURE_IS_X86})
add_memset(memset_x86_64_opt_sse4 COMPILE_OPTIONS -march=nehalem REQUIRE SSE4_2)
add_memset(memset_x86_64_opt_avx2 COMPILE_OPTIONS -march=haswell REQUIRE AVX2)
add_memset(memset_x86_64_opt_avx512 COMPILE_OPTIONS -march=skylake-avx512 REQUIRE AVX512F)
+ add_memset(memset_x86_64_opt_sw_prefetch COMPILE_OPTIONS -DLIBC_COPT_MEMSET_X86_USE_SOFTWARE_PREFETCHING)
add_memset(memset_opt_host COMPILE_OPTIONS ${LIBC_COMPILE_OPTIONS_NATIVE})
add_memset(memset)
elseif(${LIBC_TARGET_ARCHITECTURE_IS_AARCH64})
diff --git a/libc/src/string/memory_utils/x86_64/inline_memset.h b/libc/src/string/memory_utils/x86_64/inline_memset.h
index da463bc0029f9aa..f3ad04930c52c64 100644
--- a/libc/src/string/memory_utils/x86_64/inline_memset.h
+++ b/libc/src/string/memory_utils/x86_64/inline_memset.h
@@ -16,12 +16,34 @@
#include <stddef.h> // size_t
namespace LIBC_NAMESPACE {
+namespace x86 {
static constexpr size_t kCachelineSize = 64;
+LIBC_INLINE_VAR constexpr bool kUseSoftwarePrefetching =
+ LLVM_LIBC_IS_DEFINED(LIBC_COPT_MEMSET_X86_USE_SOFTWARE_PREFETCHING);
+
+} // namespace x86
+
// prefetch for write
static inline void PrefetchW(CPtr dst) { __builtin_prefetch(dst, 1, 3); }
+[[maybe_unused]] LIBC_INLINE static void inline_memset_x86_sw_prefetching(Ptr dst, uint8_t value, size_t count) {
+ PrefetchW(dst + kCachelineSize);
+ if (count <= 128)
+ return generic::Memset<uint512_t>::head_tail(dst, value, count);
+ PrefetchW(dst + kCachelineSize * 2);
+ // Aligned loop
+ generic::Memset<uint256_t>::block(dst, value);
+ align_to_next_boundary<32>(dst, count);
+ if (count <= 192) {
+ return Memset<uint256_t>::loop_and_tail(dst, value, count);
+ }
+ else {
+ return Memset<uint256_t>::loop_and_tail_prefetch<320, 128>(dst, value, count);
+ }
+}
+
[[maybe_unused]] LIBC_INLINE static void
inline_memset_x86(Ptr dst, uint8_t value, size_t count) {
#if defined(__AVX512F__)
@@ -58,17 +80,17 @@ inline_memset_x86(Ptr dst, uint8_t value, size_t count) {
return generic::Memset<uint128_t>::head_tail(dst, value, count);
if (count <= 64)
return generic::Memset<uint256_t>::head_tail(dst, value, count);
- PrefetchW(dst + kCachelineSize);
+ if constexpr (x86::kUseSoftwarePrefetching) {
+ return inline_memset_x86_sw_prefetching(dst, value, count);
+ }
if (count <= 128)
return generic::Memset<uint512_t>::head_tail(dst, value, count);
- PrefetchW(dst + kCachelineSize * 2);
// Aligned loop
generic::Memset<uint256_t>::block(dst, value);
align_to_next_boundary<32>(dst, count);
- if (count <= 192) {
- return Memset<uint256_t>::loop_and_tail(dst, value, count);
+ else {
+ return Memset<uint256_t>::loop_and_tail(dst, value, count);
}
- return Memset<uint256_t>::loop_and_tail_prefetch<320, 128>(dst, value, count);
}
} // namespace LIBC_NAMESPACE
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 3ae68193dccd2b2..dea21fd77182605 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -33,6 +33,7 @@ PRINTF_COPTS = [
MEMORY_COPTS = [
# "LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE=0",
# "LIBC_COPT_MEMCPY_X86_USE_SOFTWARE_PREFETCHING",
+ # "LIBC_COPT_MEMSET_X86_USE_SOFTWARE_PREFETCHING",
]
# A flag to pick which `mpfr` to use for math tests.
>From abb9debc49b7e171eae14a98320b9a49779c808c Mon Sep 17 00:00:00 2001
From: Your Name <milidoshi at google.com>
Date: Fri, 27 Oct 2023 17:55:47 +0000
Subject: [PATCH 03/10] Fix formatting
---
libc/src/string/memory_utils/x86_64/inline_memset.h | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/libc/src/string/memory_utils/x86_64/inline_memset.h b/libc/src/string/memory_utils/x86_64/inline_memset.h
index f3ad04930c52c64..e82b600bf66ab96 100644
--- a/libc/src/string/memory_utils/x86_64/inline_memset.h
+++ b/libc/src/string/memory_utils/x86_64/inline_memset.h
@@ -28,7 +28,8 @@ LIBC_INLINE_VAR constexpr bool kUseSoftwarePrefetching =
// prefetch for write
static inline void PrefetchW(CPtr dst) { __builtin_prefetch(dst, 1, 3); }
-[[maybe_unused]] LIBC_INLINE static void inline_memset_x86_sw_prefetching(Ptr dst, uint8_t value, size_t count) {
+[[maybe_unused]] LIBC_INLINE static void
+inline_memset_x86_sw_prefetching(Ptr dst, uint8_t value, size_t count) {
PrefetchW(dst + kCachelineSize);
if (count <= 128)
return generic::Memset<uint512_t>::head_tail(dst, value, count);
@@ -38,9 +39,9 @@ static inline void PrefetchW(CPtr dst) { __builtin_prefetch(dst, 1, 3); }
align_to_next_boundary<32>(dst, count);
if (count <= 192) {
return Memset<uint256_t>::loop_and_tail(dst, value, count);
- }
- else {
- return Memset<uint256_t>::loop_and_tail_prefetch<320, 128>(dst, value, count);
+ } else {
+ return Memset<uint256_t>::loop_and_tail_prefetch<320, 128>(dst, value,
+ count);
}
}
@@ -89,7 +90,7 @@ inline_memset_x86(Ptr dst, uint8_t value, size_t count) {
generic::Memset<uint256_t>::block(dst, value);
align_to_next_boundary<32>(dst, count);
else {
- return Memset<uint256_t>::loop_and_tail(dst, value, count);
+ return Memset<uint256_t>::loop_and_tail(dst, value, count);
}
}
} // namespace LIBC_NAMESPACE
>From 2155db70066c2c220160c4178bd73237e1372d45 Mon Sep 17 00:00:00 2001
From: Your Name <milidoshi at google.com>
Date: Mon, 30 Oct 2023 14:53:56 +0000
Subject: [PATCH 04/10] Fix build errors
---
libc/src/string/memory_utils/op_generic.h | 15 +++++++++------
.../string/memory_utils/x86_64/inline_memset.h | 7 ++-----
2 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/libc/src/string/memory_utils/op_generic.h b/libc/src/string/memory_utils/op_generic.h
index 54af7ea10e25e46..4ba137c97ec9a9a 100644
--- a/libc/src/string/memory_utils/op_generic.h
+++ b/libc/src/string/memory_utils/op_generic.h
@@ -87,6 +87,9 @@ template <class T, size_t N>
struct array_size<cpp::array<T, N>> : cpp::integral_constant<size_t, N> {};
template <typename T> constexpr size_t array_size_v = array_size<T>::value;
+// Size of a cacheline for software prefetching
+static constexpr size_t kCachelineSize = 64;
+
// Generic operations for the above type categories.
template <typename T> T load(CPtr src) {
@@ -167,18 +170,18 @@ template <typename T> struct Memset {
template <size_t prefetch_distance, size_t prefetch_degree>
LIBC_INLINE static void loop_and_tail_prefetch(Ptr dst, uint8_t value,
size_t count) {
- Memset<uint512_t>::block(dst, value);
- Memset<uint256_t>::block(dst + 64, value);
+ Memset<64>::block(dst, value);
+ Memset<32>::block(dst + 64, value);
size_t offset = 96;
- while (offset + prefetch_degree + kSize <= count) {
+ while (offset + prefetch_degree + SIZE <= count) {
for (size_t i = 0; i < prefetch_degree / kCachelineSize; ++i)
PrefetchW(dst + offset + prefetch_distance + kCachelineSize * i);
- for (size_t i = 0; i < prefetch_degree; i += kSize, offset += kSize)
+ for (size_t i = 0; i < prefetch_degree; i += SIZE, offset += SIZE)
block(dst + offset, value);
}
- while (offset + kSize < count) {
+ while (offset + SIZE < count) {
block(dst + offset, value);
- offset += kSize;
+ offset += SIZE;
}
tail(dst, value, count);
}
diff --git a/libc/src/string/memory_utils/x86_64/inline_memset.h b/libc/src/string/memory_utils/x86_64/inline_memset.h
index e82b600bf66ab96..fca48e9658a752d 100644
--- a/libc/src/string/memory_utils/x86_64/inline_memset.h
+++ b/libc/src/string/memory_utils/x86_64/inline_memset.h
@@ -17,9 +17,6 @@
namespace LIBC_NAMESPACE {
namespace x86 {
-
-static constexpr size_t kCachelineSize = 64;
-
LIBC_INLINE_VAR constexpr bool kUseSoftwarePrefetching =
LLVM_LIBC_IS_DEFINED(LIBC_COPT_MEMSET_X86_USE_SOFTWARE_PREFETCHING);
@@ -30,10 +27,10 @@ static inline void PrefetchW(CPtr dst) { __builtin_prefetch(dst, 1, 3); }
[[maybe_unused]] LIBC_INLINE static void
inline_memset_x86_sw_prefetching(Ptr dst, uint8_t value, size_t count) {
- PrefetchW(dst + kCachelineSize);
+ PrefetchW(dst + generic::kCachelineSize);
if (count <= 128)
return generic::Memset<uint512_t>::head_tail(dst, value, count);
- PrefetchW(dst + kCachelineSize * 2);
+ PrefetchW(dst + generic::kCachelineSize * 2);
// Aligned loop
generic::Memset<uint256_t>::block(dst, value);
align_to_next_boundary<32>(dst, count);
>From 9fe0041c2bb8ba1d522538c79ac1ebae7d0632bb Mon Sep 17 00:00:00 2001
From: Your Name <milidoshi at google.com>
Date: Mon, 30 Oct 2023 17:09:15 +0000
Subject: [PATCH 05/10] Fix build errors
---
libc/src/string/memory_utils/op_generic.h | 15 +++---
.../memory_utils/x86_64/inline_memset.h | 50 +++++++++----------
2 files changed, 33 insertions(+), 32 deletions(-)
diff --git a/libc/src/string/memory_utils/op_generic.h b/libc/src/string/memory_utils/op_generic.h
index 4ba137c97ec9a9a..12eeb65a1edc52e 100644
--- a/libc/src/string/memory_utils/op_generic.h
+++ b/libc/src/string/memory_utils/op_generic.h
@@ -48,6 +48,13 @@ using generic_v256 = uint8_t __attribute__((__vector_size__(32)));
using generic_v512 = uint8_t __attribute__((__vector_size__(64)));
} // namespace LIBC_NAMESPACE
+namespace sw_prefetch {
+ // Size of a cacheline for software prefetching
+static constexpr size_t kCachelineSize = 64;
+ // prefetch for write
+static inline void PrefetchW(CPtr dst) { __builtin_prefetch(dst, 1, 3); }
+}
+
namespace LIBC_NAMESPACE::generic {
// We accept three types of values as elements for generic operations:
@@ -87,9 +94,6 @@ template <class T, size_t N>
struct array_size<cpp::array<T, N>> : cpp::integral_constant<size_t, N> {};
template <typename T> constexpr size_t array_size_v = array_size<T>::value;
-// Size of a cacheline for software prefetching
-static constexpr size_t kCachelineSize = 64;
-
// Generic operations for the above type categories.
template <typename T> T load(CPtr src) {
@@ -167,12 +171,9 @@ template <typename T> struct Memset {
tail(dst, value, count);
}
- template <size_t prefetch_distance, size_t prefetch_degree>
+ template <size_t prefetch_distance, size_t prefetch_degree, size_t offset>
LIBC_INLINE static void loop_and_tail_prefetch(Ptr dst, uint8_t value,
size_t count) {
- Memset<64>::block(dst, value);
- Memset<32>::block(dst + 64, value);
- size_t offset = 96;
while (offset + prefetch_degree + SIZE <= count) {
for (size_t i = 0; i < prefetch_degree / kCachelineSize; ++i)
PrefetchW(dst + offset + prefetch_distance + kCachelineSize * i);
diff --git a/libc/src/string/memory_utils/x86_64/inline_memset.h b/libc/src/string/memory_utils/x86_64/inline_memset.h
index fca48e9658a752d..bc7a6162f77b9cd 100644
--- a/libc/src/string/memory_utils/x86_64/inline_memset.h
+++ b/libc/src/string/memory_utils/x86_64/inline_memset.h
@@ -17,33 +17,11 @@
namespace LIBC_NAMESPACE {
namespace x86 {
-LIBC_INLINE_VAR constexpr bool kUseSoftwarePrefetching =
+LIBC_INLINE_VAR constexpr bool kUseSoftwarePrefetchingMemset =
LLVM_LIBC_IS_DEFINED(LIBC_COPT_MEMSET_X86_USE_SOFTWARE_PREFETCHING);
} // namespace x86
-// prefetch for write
-static inline void PrefetchW(CPtr dst) { __builtin_prefetch(dst, 1, 3); }
-
-[[maybe_unused]] LIBC_INLINE static void
-inline_memset_x86_sw_prefetching(Ptr dst, uint8_t value, size_t count) {
- PrefetchW(dst + generic::kCachelineSize);
- if (count <= 128)
- return generic::Memset<uint512_t>::head_tail(dst, value, count);
- PrefetchW(dst + generic::kCachelineSize * 2);
- // Aligned loop
- generic::Memset<uint256_t>::block(dst, value);
- align_to_next_boundary<32>(dst, count);
- if (count <= 192) {
- return Memset<uint256_t>::loop_and_tail(dst, value, count);
- } else {
- return Memset<uint256_t>::loop_and_tail_prefetch<320, 128>(dst, value,
- count);
- }
-}
-
-[[maybe_unused]] LIBC_INLINE static void
-inline_memset_x86(Ptr dst, uint8_t value, size_t count) {
#if defined(__AVX512F__)
using uint128_t = generic_v128;
using uint256_t = generic_v256;
@@ -62,6 +40,28 @@ inline_memset_x86(Ptr dst, uint8_t value, size_t count) {
using uint512_t = cpp::array<uint64_t, 8>;
#endif
+[[maybe_unused]] LIBC_INLINE static void
+inline_memset_x86_sw_prefetching(Ptr dst, uint8_t value, size_t count) {
+ sw_prefetch::PrefetchW(dst + generic::kCachelineSize);
+ if (count <= 128)
+ return generic::Memset<uint512_t>::head_tail(dst, value, count);
+ sw_prefetch::PrefetchW(dst + generic::kCachelineSize * 2);
+ // Aligned loop
+ generic::Memset<uint256_t>::block(dst, value);
+ align_to_next_boundary<32>(dst, count);
+ if (count <= 192) {
+ return Memset<uint256_t>::loop_and_tail(dst, value, count);
+ } else {
+ // Warm up memset
+ generic::Memset<uint256_t>::block(dst, value);
+ generic::Memset<uint128_t>::block(dst + 64, value);
+ return generic::Memset<uint256_t>::loop_and_tail_prefetch<320, 128, 96>(dst, value,
+ count);
+ }
+}
+
+[[maybe_unused]] LIBC_INLINE static void
+inline_memset_x86(Ptr dst, uint8_t value, size_t count) {
if (count == 0)
return;
if (count == 1)
@@ -78,7 +78,7 @@ inline_memset_x86(Ptr dst, uint8_t value, size_t count) {
return generic::Memset<uint128_t>::head_tail(dst, value, count);
if (count <= 64)
return generic::Memset<uint256_t>::head_tail(dst, value, count);
- if constexpr (x86::kUseSoftwarePrefetching) {
+ if constexpr (x86::kUseSoftwarePrefetchingMemset) {
return inline_memset_x86_sw_prefetching(dst, value, count);
}
if (count <= 128)
@@ -87,7 +87,7 @@ inline_memset_x86(Ptr dst, uint8_t value, size_t count) {
generic::Memset<uint256_t>::block(dst, value);
align_to_next_boundary<32>(dst, count);
else {
- return Memset<uint256_t>::loop_and_tail(dst, value, count);
+ return generic::Memset<uint256_t>::loop_and_tail(dst, value, count);
}
}
} // namespace LIBC_NAMESPACE
>From 52aad858a4a8652f95a3e1120e4dd7bd2f45d225 Mon Sep 17 00:00:00 2001
From: Your Name <milidoshi at google.com>
Date: Mon, 30 Oct 2023 17:39:04 +0000
Subject: [PATCH 06/10] Fix formatting
---
libc/src/string/memory_utils/op_generic.h | 6 +-
.../memory_utils/x86_64/inline_memset.h | 92 +++++++++----------
2 files changed, 49 insertions(+), 49 deletions(-)
diff --git a/libc/src/string/memory_utils/op_generic.h b/libc/src/string/memory_utils/op_generic.h
index 12eeb65a1edc52e..af6a814be1542a4 100644
--- a/libc/src/string/memory_utils/op_generic.h
+++ b/libc/src/string/memory_utils/op_generic.h
@@ -49,11 +49,11 @@ using generic_v512 = uint8_t __attribute__((__vector_size__(64)));
} // namespace LIBC_NAMESPACE
namespace sw_prefetch {
- // Size of a cacheline for software prefetching
+// Size of a cacheline for software prefetching
static constexpr size_t kCachelineSize = 64;
- // prefetch for write
+// prefetch for write
static inline void PrefetchW(CPtr dst) { __builtin_prefetch(dst, 1, 3); }
-}
+} // namespace sw_prefetch
namespace LIBC_NAMESPACE::generic {
diff --git a/libc/src/string/memory_utils/x86_64/inline_memset.h b/libc/src/string/memory_utils/x86_64/inline_memset.h
index bc7a6162f77b9cd..9000aa03019d291 100644
--- a/libc/src/string/memory_utils/x86_64/inline_memset.h
+++ b/libc/src/string/memory_utils/x86_64/inline_memset.h
@@ -40,56 +40,56 @@ LIBC_INLINE_VAR constexpr bool kUseSoftwarePrefetchingMemset =
using uint512_t = cpp::array<uint64_t, 8>;
#endif
-[[maybe_unused]] LIBC_INLINE static void
-inline_memset_x86_sw_prefetching(Ptr dst, uint8_t value, size_t count) {
- sw_prefetch::PrefetchW(dst + generic::kCachelineSize);
- if (count <= 128)
- return generic::Memset<uint512_t>::head_tail(dst, value, count);
- sw_prefetch::PrefetchW(dst + generic::kCachelineSize * 2);
- // Aligned loop
- generic::Memset<uint256_t>::block(dst, value);
- align_to_next_boundary<32>(dst, count);
- if (count <= 192) {
- return Memset<uint256_t>::loop_and_tail(dst, value, count);
- } else {
- // Warm up memset
+ [[maybe_unused]] LIBC_INLINE static void
+ inline_memset_x86_sw_prefetching(Ptr dst, uint8_t value, size_t count) {
+ sw_prefetch::PrefetchW(dst + generic::kCachelineSize);
+ if (count <= 128)
+ return generic::Memset<uint512_t>::head_tail(dst, value, count);
+ sw_prefetch::PrefetchW(dst + generic::kCachelineSize * 2);
+ // Aligned loop
generic::Memset<uint256_t>::block(dst, value);
- generic::Memset<uint128_t>::block(dst + 64, value);
- return generic::Memset<uint256_t>::loop_and_tail_prefetch<320, 128, 96>(dst, value,
- count);
+ align_to_next_boundary<32>(dst, count);
+ if (count <= 192) {
+ return Memset<uint256_t>::loop_and_tail(dst, value, count);
+ } else {
+ // Warm up memset
+ generic::Memset<uint256_t>::block(dst, value);
+ generic::Memset<uint128_t>::block(dst + 64, value);
+ return generic::Memset<uint256_t>::loop_and_tail_prefetch<320, 128, 96>(
+ dst, value, count);
+ }
}
-}
-[[maybe_unused]] LIBC_INLINE static void
-inline_memset_x86(Ptr dst, uint8_t value, size_t count) {
- if (count == 0)
- return;
- if (count == 1)
- return generic::Memset<uint8_t>::block(dst, value);
- if (count == 2)
- return generic::Memset<uint16_t>::block(dst, value);
- if (count == 3)
- return generic::MemsetSequence<uint16_t, uint8_t>::block(dst, value);
- if (count <= 8)
- return generic::Memset<uint32_t>::head_tail(dst, value, count);
- if (count <= 16)
- return generic::Memset<uint64_t>::head_tail(dst, value, count);
- if (count <= 32)
- return generic::Memset<uint128_t>::head_tail(dst, value, count);
- if (count <= 64)
- return generic::Memset<uint256_t>::head_tail(dst, value, count);
- if constexpr (x86::kUseSoftwarePrefetchingMemset) {
- return inline_memset_x86_sw_prefetching(dst, value, count);
- }
- if (count <= 128)
- return generic::Memset<uint512_t>::head_tail(dst, value, count);
- // Aligned loop
- generic::Memset<uint256_t>::block(dst, value);
- align_to_next_boundary<32>(dst, count);
- else {
- return generic::Memset<uint256_t>::loop_and_tail(dst, value, count);
+ [[maybe_unused]] LIBC_INLINE static void
+ inline_memset_x86(Ptr dst, uint8_t value, size_t count) {
+ if (count == 0)
+ return;
+ if (count == 1)
+ return generic::Memset<uint8_t>::block(dst, value);
+ if (count == 2)
+ return generic::Memset<uint16_t>::block(dst, value);
+ if (count == 3)
+ return generic::MemsetSequence<uint16_t, uint8_t>::block(dst, value);
+ if (count <= 8)
+ return generic::Memset<uint32_t>::head_tail(dst, value, count);
+ if (count <= 16)
+ return generic::Memset<uint64_t>::head_tail(dst, value, count);
+ if (count <= 32)
+ return generic::Memset<uint128_t>::head_tail(dst, value, count);
+ if (count <= 64)
+ return generic::Memset<uint256_t>::head_tail(dst, value, count);
+ if constexpr (x86::kUseSoftwarePrefetchingMemset) {
+ return inline_memset_x86_sw_prefetching(dst, value, count);
+ }
+ if (count <= 128)
+ return generic::Memset<uint512_t>::head_tail(dst, value, count);
+ // Aligned loop
+ generic::Memset<uint256_t>::block(dst, value);
+ align_to_next_boundary<32>(dst, count);
+ else {
+ return generic::Memset<uint256_t>::loop_and_tail(dst, value, count);
+ }
}
-}
} // namespace LIBC_NAMESPACE
#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_MEMSET_H
>From efbfcd19cecbd3e27b72523d06d8cff3a5bbbafa Mon Sep 17 00:00:00 2001
From: Your Name <milidoshi at google.com>
Date: Mon, 30 Oct 2023 19:12:19 +0000
Subject: [PATCH 07/10] Fix formatting
---
libc/src/string/memory_utils/op_generic.h | 24 +++++++++++--------
.../memory_utils/x86_64/inline_memset.h | 6 ++---
2 files changed, 17 insertions(+), 13 deletions(-)
diff --git a/libc/src/string/memory_utils/op_generic.h b/libc/src/string/memory_utils/op_generic.h
index af6a814be1542a4..ae221e0fa380655 100644
--- a/libc/src/string/memory_utils/op_generic.h
+++ b/libc/src/string/memory_utils/op_generic.h
@@ -48,12 +48,12 @@ using generic_v256 = uint8_t __attribute__((__vector_size__(32)));
using generic_v512 = uint8_t __attribute__((__vector_size__(64)));
} // namespace LIBC_NAMESPACE
-namespace sw_prefetch {
+namespace LIBC_NAMESPACE::sw_prefetch {
// Size of a cacheline for software prefetching
static constexpr size_t kCachelineSize = 64;
// prefetch for write
static inline void PrefetchW(CPtr dst) { __builtin_prefetch(dst, 1, 3); }
-} // namespace sw_prefetch
+} // namespace LIBC_NAMESPACE::sw_prefetch
namespace LIBC_NAMESPACE::generic {
@@ -174,15 +174,19 @@ template <typename T> struct Memset {
template <size_t prefetch_distance, size_t prefetch_degree, size_t offset>
LIBC_INLINE static void loop_and_tail_prefetch(Ptr dst, uint8_t value,
size_t count) {
- while (offset + prefetch_degree + SIZE <= count) {
- for (size_t i = 0; i < prefetch_degree / kCachelineSize; ++i)
- PrefetchW(dst + offset + prefetch_distance + kCachelineSize * i);
- for (size_t i = 0; i < prefetch_degree; i += SIZE, offset += SIZE)
- block(dst + offset, value);
+ size_t prefetch_offset = offset;
+
+ while (prefetch_offset + prefetch_degree + SIZE <= count) {
+ for (size_t i = 0; i < prefetch_degree / sw_prefetch::kCachelineSize; ++i)
+ PrefetchW(dst + prefetch_offset + prefetch_distance +
+ sw_prefetch::kCachelineSize * i);
+ for (size_t i = 0; i < prefetch_degree;
+ i += SIZE, prefetch_offset += SIZE)
+ block(dst + prefetch_offset, value);
}
- while (offset + SIZE < count) {
- block(dst + offset, value);
- offset += SIZE;
+ while (prefetch_offset + SIZE < count) {
+ block(dst + prefetch_offset, value);
+ prefetch_offset += SIZE;
}
tail(dst, value, count);
}
diff --git a/libc/src/string/memory_utils/x86_64/inline_memset.h b/libc/src/string/memory_utils/x86_64/inline_memset.h
index 9000aa03019d291..fc00f86fc0fb34e 100644
--- a/libc/src/string/memory_utils/x86_64/inline_memset.h
+++ b/libc/src/string/memory_utils/x86_64/inline_memset.h
@@ -42,15 +42,15 @@ LIBC_INLINE_VAR constexpr bool kUseSoftwarePrefetchingMemset =
[[maybe_unused]] LIBC_INLINE static void
inline_memset_x86_sw_prefetching(Ptr dst, uint8_t value, size_t count) {
- sw_prefetch::PrefetchW(dst + generic::kCachelineSize);
+ sw_prefetch::PrefetchW(dst + sw_prefetch::kCachelineSize);
if (count <= 128)
return generic::Memset<uint512_t>::head_tail(dst, value, count);
- sw_prefetch::PrefetchW(dst + generic::kCachelineSize * 2);
+ sw_prefetch::PrefetchW(dst + sw_prefetch::kCachelineSize * 2);
// Aligned loop
generic::Memset<uint256_t>::block(dst, value);
align_to_next_boundary<32>(dst, count);
if (count <= 192) {
- return Memset<uint256_t>::loop_and_tail(dst, value, count);
+ return generic::Memset<uint256_t>::loop_and_tail(dst, value, count);
} else {
// Warm up memset
generic::Memset<uint256_t>::block(dst, value);
>From d97c8c0d17c860a892c029059469b70962e4a201 Mon Sep 17 00:00:00 2001
From: Your Name <milidoshi at google.com>
Date: Mon, 30 Oct 2023 20:21:49 +0000
Subject: [PATCH 08/10] Fix formatting
---
libc/src/string/memory_utils/op_generic.h | 4 ++--
libc/src/string/memory_utils/x86_64/inline_memset.h | 4 +---
2 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/libc/src/string/memory_utils/op_generic.h b/libc/src/string/memory_utils/op_generic.h
index ae221e0fa380655..35f74b544bb3598 100644
--- a/libc/src/string/memory_utils/op_generic.h
+++ b/libc/src/string/memory_utils/op_generic.h
@@ -178,8 +178,8 @@ template <typename T> struct Memset {
while (prefetch_offset + prefetch_degree + SIZE <= count) {
for (size_t i = 0; i < prefetch_degree / sw_prefetch::kCachelineSize; ++i)
- PrefetchW(dst + prefetch_offset + prefetch_distance +
- sw_prefetch::kCachelineSize * i);
+ sw_prefetch::PrefetchW(dst + prefetch_offset + prefetch_distance +
+ sw_prefetch::kCachelineSize * i);
for (size_t i = 0; i < prefetch_degree;
i += SIZE, prefetch_offset += SIZE)
block(dst + prefetch_offset, value);
diff --git a/libc/src/string/memory_utils/x86_64/inline_memset.h b/libc/src/string/memory_utils/x86_64/inline_memset.h
index fc00f86fc0fb34e..50ba2fb2e37cfab 100644
--- a/libc/src/string/memory_utils/x86_64/inline_memset.h
+++ b/libc/src/string/memory_utils/x86_64/inline_memset.h
@@ -86,9 +86,7 @@ LIBC_INLINE_VAR constexpr bool kUseSoftwarePrefetchingMemset =
// Aligned loop
generic::Memset<uint256_t>::block(dst, value);
align_to_next_boundary<32>(dst, count);
- else {
- return generic::Memset<uint256_t>::loop_and_tail(dst, value, count);
- }
+ return generic::Memset<uint256_t>::loop_and_tail(dst, value, count);
}
} // namespace LIBC_NAMESPACE
>From 34d572e81b561b4450022dc358f6e3a91632224f Mon Sep 17 00:00:00 2001
From: Your Name <milidoshi at google.com>
Date: Mon, 30 Oct 2023 20:21:49 +0000
Subject: [PATCH 09/10] Fix formatting
---
libc/src/string/memory_utils/op_generic.h | 18 +++++++++---------
.../string/memory_utils/x86_64/inline_memset.h | 5 +----
2 files changed, 10 insertions(+), 13 deletions(-)
diff --git a/libc/src/string/memory_utils/op_generic.h b/libc/src/string/memory_utils/op_generic.h
index 35f74b544bb3598..f36c3acafff5665 100644
--- a/libc/src/string/memory_utils/op_generic.h
+++ b/libc/src/string/memory_utils/op_generic.h
@@ -171,22 +171,22 @@ template <typename T> struct Memset {
tail(dst, value, count);
}
- template <size_t prefetch_distance, size_t prefetch_degree, size_t offset>
+ template <size_t prefetch_distance, size_t prefetch_degree>
LIBC_INLINE static void loop_and_tail_prefetch(Ptr dst, uint8_t value,
size_t count) {
- size_t prefetch_offset = offset;
+ size_t offset = 0;
- while (prefetch_offset + prefetch_degree + SIZE <= count) {
+ while (offset + prefetch_degree + SIZE <= count) {
for (size_t i = 0; i < prefetch_degree / sw_prefetch::kCachelineSize; ++i)
- sw_prefetch::PrefetchW(dst + prefetch_offset + prefetch_distance +
+ sw_prefetch::PrefetchW(dst + offset + prefetch_distance +
sw_prefetch::kCachelineSize * i);
for (size_t i = 0; i < prefetch_degree;
- i += SIZE, prefetch_offset += SIZE)
- block(dst + prefetch_offset, value);
+ i += SIZE, offset += SIZE)
+ block(dst + offset, value);
}
- while (prefetch_offset + SIZE < count) {
- block(dst + prefetch_offset, value);
- prefetch_offset += SIZE;
+ while (offset + SIZE < count) {
+ block(dst + offset, value);
+ offset += SIZE;
}
tail(dst, value, count);
}
diff --git a/libc/src/string/memory_utils/x86_64/inline_memset.h b/libc/src/string/memory_utils/x86_64/inline_memset.h
index 50ba2fb2e37cfab..4834968c0b99f38 100644
--- a/libc/src/string/memory_utils/x86_64/inline_memset.h
+++ b/libc/src/string/memory_utils/x86_64/inline_memset.h
@@ -52,10 +52,7 @@ LIBC_INLINE_VAR constexpr bool kUseSoftwarePrefetchingMemset =
if (count <= 192) {
return generic::Memset<uint256_t>::loop_and_tail(dst, value, count);
} else {
- // Warm up memset
- generic::Memset<uint256_t>::block(dst, value);
- generic::Memset<uint128_t>::block(dst + 64, value);
- return generic::Memset<uint256_t>::loop_and_tail_prefetch<320, 128, 96>(
+ return generic::Memset<uint256_t>::loop_and_tail_prefetch<320, 128>(
dst, value, count);
}
}
>From f363ce21eedc44821e5163ad9472396856a096c1 Mon Sep 17 00:00:00 2001
From: Your Name <milidoshi at google.com>
Date: Mon, 30 Oct 2023 20:21:49 +0000
Subject: [PATCH 10/10] Fix formatting
---
libc/src/string/memory_utils/op_generic.h | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/libc/src/string/memory_utils/op_generic.h b/libc/src/string/memory_utils/op_generic.h
index f36c3acafff5665..4063de1d5f5832a 100644
--- a/libc/src/string/memory_utils/op_generic.h
+++ b/libc/src/string/memory_utils/op_generic.h
@@ -180,8 +180,7 @@ template <typename T> struct Memset {
for (size_t i = 0; i < prefetch_degree / sw_prefetch::kCachelineSize; ++i)
sw_prefetch::PrefetchW(dst + offset + prefetch_distance +
sw_prefetch::kCachelineSize * i);
- for (size_t i = 0; i < prefetch_degree;
- i += SIZE, offset += SIZE)
+ for (size_t i = 0; i < prefetch_degree; i += SIZE, offset += SIZE)
block(dst + offset, value);
}
while (offset + SIZE < count) {
More information about the libc-commits
mailing list