[libc-commits] [libc] 9a8d64a - Revert "[libc] Separate memcpy implementations per arch"
Guillaume Chatelet via libc-commits
libc-commits at lists.llvm.org
Tue Feb 14 08:15:28 PST 2023
Author: Guillaume Chatelet
Date: 2023-02-14T16:14:51Z
New Revision: 9a8d64a66a15327e210d68d78984da972323cc71
URL: https://github.com/llvm/llvm-project/commit/9a8d64a66a15327e210d68d78984da972323cc71
DIFF: https://github.com/llvm/llvm-project/commit/9a8d64a66a15327e210d68d78984da972323cc71.diff
LOG: Revert "[libc] Separate memcpy implementations per arch"
Reverting as it broke downstream users.
This reverts commit 26541d4176346f8510302c74d6f3b958a4e523db.
Added:
Modified:
libc/src/string/memory_utils/CMakeLists.txt
libc/src/string/memory_utils/memcpy_implementations.h
utils/bazel/llvm-project-overlay/libc/BUILD.bazel
Removed:
libc/src/string/memory_utils/aarch64/memcpy_implementations.h
libc/src/string/memory_utils/x86_64/memcpy_implementations.h
################################################################################
diff --git a/libc/src/string/memory_utils/CMakeLists.txt b/libc/src/string/memory_utils/CMakeLists.txt
index a1cbff0df1e32..371b4010a4dc8 100644
--- a/libc/src/string/memory_utils/CMakeLists.txt
+++ b/libc/src/string/memory_utils/CMakeLists.txt
@@ -2,7 +2,6 @@
add_header_library(
memory_utils
HDRS
- aarch64/memcpy_implementations.h
bcmp_implementations.h
bzero_implementations.h
memcmp_implementations.h
@@ -14,12 +13,11 @@ add_header_library(
op_generic.h
op_x86.h
utils.h
- x86_64/memcpy_implementations.h
DEPS
+ libc.src.__support.common
libc.src.__support.CPP.bit
libc.src.__support.CPP.cstddef
libc.src.__support.CPP.type_traits
- libc.src.__support.macros.config
libc.src.__support.macros.optimization
)
diff --git a/libc/src/string/memory_utils/aarch64/memcpy_implementations.h b/libc/src/string/memory_utils/aarch64/memcpy_implementations.h
deleted file mode 100644
index e34eba27ece5b..0000000000000
--- a/libc/src/string/memory_utils/aarch64/memcpy_implementations.h
+++ /dev/null
@@ -1,48 +0,0 @@
-//===-- Memcpy implementation for aarch64 -----------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-#ifndef LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMCPY_IMPLEMENTATIONS_H
-#define LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMCPY_IMPLEMENTATIONS_H
-
-#include "src/__support/macros/config.h" // LIBC_INLINE
-#include "src/string/memory_utils/op_builtin.h"
-#include "src/string/memory_utils/utils.h"
-
-#include <stddef.h> // size_t
-
-namespace __llvm_libc {
-
-[[maybe_unused]] LIBC_INLINE void
-inline_memcpy_aarch64(Ptr __restrict dst, CPtr __restrict src, size_t count) {
- if (count == 0)
- return;
- if (count == 1)
- return builtin::Memcpy<1>::block(dst, src);
- if (count == 2)
- return builtin::Memcpy<2>::block(dst, src);
- if (count == 3)
- return builtin::Memcpy<3>::block(dst, src);
- if (count == 4)
- return builtin::Memcpy<4>::block(dst, src);
- if (count < 8)
- return builtin::Memcpy<4>::head_tail(dst, src, count);
- if (count < 16)
- return builtin::Memcpy<8>::head_tail(dst, src, count);
- if (count < 32)
- return builtin::Memcpy<16>::head_tail(dst, src, count);
- if (count < 64)
- return builtin::Memcpy<32>::head_tail(dst, src, count);
- if (count < 128)
- return builtin::Memcpy<64>::head_tail(dst, src, count);
- builtin::Memcpy<16>::block(dst, src);
- align_to_next_boundary<16, Arg::Src>(dst, src, count);
- return builtin::Memcpy<64>::loop_and_tail(dst, src, count);
-}
-
-} // namespace __llvm_libc
-
-#endif // LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_MEMCPY_IMPLEMENTATIONS_H
diff --git a/libc/src/string/memory_utils/memcpy_implementations.h b/libc/src/string/memory_utils/memcpy_implementations.h
index ea5c43bebbbdf..850e0256c7b1d 100644
--- a/libc/src/string/memory_utils/memcpy_implementations.h
+++ b/libc/src/string/memory_utils/memcpy_implementations.h
@@ -9,19 +9,17 @@
#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H
#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H
-#include "src/__support/macros/config.h" // LIBC_INLINE
-#include "src/__support/macros/optimization.h" // LIBC_LOOP_NOUNROLL
+#include "src/__support/common.h"
+#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY LIBC_LOOP_NOUNROLL
#include "src/__support/macros/properties/architectures.h"
+#include "src/string/memory_utils/op_aarch64.h"
+#include "src/string/memory_utils/op_builtin.h"
+#include "src/string/memory_utils/op_generic.h"
+#include "src/string/memory_utils/op_x86.h"
#include "src/string/memory_utils/utils.h"
#include <stddef.h> // size_t
-#if defined(LIBC_TARGET_ARCH_IS_X86)
-#include "src/string/memory_utils/x86_64/memcpy_implementations.h"
-#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
-#include "src/string/memory_utils/aarch64/memcpy_implementations.h"
-#endif
-
namespace __llvm_libc {
[[maybe_unused]] LIBC_INLINE void
@@ -32,6 +30,98 @@ inline_memcpy_embedded_tiny(Ptr __restrict dst, CPtr __restrict src,
builtin::Memcpy<1>::block(dst + offset, src + offset);
}
+#if defined(LIBC_TARGET_ARCH_IS_X86)
+[[maybe_unused]] LIBC_INLINE void
+inline_memcpy_x86(Ptr __restrict dst, CPtr __restrict src, size_t count) {
+ if (count == 0)
+ return;
+ if (count == 1)
+ return builtin::Memcpy<1>::block(dst, src);
+ if (count == 2)
+ return builtin::Memcpy<2>::block(dst, src);
+ if (count == 3)
+ return builtin::Memcpy<3>::block(dst, src);
+ if (count == 4)
+ return builtin::Memcpy<4>::block(dst, src);
+ if (count < 8)
+ return builtin::Memcpy<4>::head_tail(dst, src, count);
+ if (count < 16)
+ return builtin::Memcpy<8>::head_tail(dst, src, count);
+ if (count < 32)
+ return builtin::Memcpy<16>::head_tail(dst, src, count);
+ if (count < 64)
+ return builtin::Memcpy<32>::head_tail(dst, src, count);
+ if (count < 128)
+ return builtin::Memcpy<64>::head_tail(dst, src, count);
+ if (x86::kAvx && count < 256)
+ return builtin::Memcpy<128>::head_tail(dst, src, count);
+ builtin::Memcpy<32>::block(dst, src);
+ align_to_next_boundary<32, Arg::Dst>(dst, src, count);
+ static constexpr size_t kBlockSize = x86::kAvx ? 64 : 32;
+ return builtin::Memcpy<kBlockSize>::loop_and_tail(dst, src, count);
+}
+
+[[maybe_unused]] LIBC_INLINE void
+inline_memcpy_x86_maybe_interpose_repmovsb(Ptr __restrict dst,
+ CPtr __restrict src, size_t count) {
+ // Whether to use rep;movsb exclusively, not at all, or only above a certain
+ // threshold.
+#ifndef LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
+#define LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE -1
+#endif
+
+#ifdef LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB
+#error LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB is deprecated use LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE=0 instead.
+#endif // LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB
+
+#ifdef LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
+#error LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE is deprecated use LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE=0 instead.
+#endif // LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
+
+ static constexpr size_t kRepMovsbThreshold =
+ LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE;
+ if constexpr (kRepMovsbThreshold == 0) {
+ return x86::Memcpy::repmovsb(dst, src, count);
+ } else if constexpr (kRepMovsbThreshold == size_t(-1)) {
+ return inline_memcpy_x86(dst, src, count);
+ } else {
+ if (LIBC_UNLIKELY(count >= kRepMovsbThreshold))
+ return x86::Memcpy::repmovsb(dst, src, count);
+ else
+ return inline_memcpy_x86(dst, src, count);
+ }
+}
+#endif // defined(LIBC_TARGET_ARCH_IS_X86)
+
+#if defined(LIBC_TARGET_ARCH_IS_AARCH64)
+[[maybe_unused]] LIBC_INLINE void
+inline_memcpy_aarch64(Ptr __restrict dst, CPtr __restrict src, size_t count) {
+ if (count == 0)
+ return;
+ if (count == 1)
+ return builtin::Memcpy<1>::block(dst, src);
+ if (count == 2)
+ return builtin::Memcpy<2>::block(dst, src);
+ if (count == 3)
+ return builtin::Memcpy<3>::block(dst, src);
+ if (count == 4)
+ return builtin::Memcpy<4>::block(dst, src);
+ if (count < 8)
+ return builtin::Memcpy<4>::head_tail(dst, src, count);
+ if (count < 16)
+ return builtin::Memcpy<8>::head_tail(dst, src, count);
+ if (count < 32)
+ return builtin::Memcpy<16>::head_tail(dst, src, count);
+ if (count < 64)
+ return builtin::Memcpy<32>::head_tail(dst, src, count);
+ if (count < 128)
+ return builtin::Memcpy<64>::head_tail(dst, src, count);
+ builtin::Memcpy<16>::block(dst, src);
+ align_to_next_boundary<16, Arg::Src>(dst, src, count);
+ return builtin::Memcpy<64>::loop_and_tail(dst, src, count);
+}
+#endif // defined(LIBC_TARGET_ARCH_IS_AARCH64)
+
LIBC_INLINE void inline_memcpy(Ptr __restrict dst, CPtr __restrict src,
size_t count) {
using namespace __llvm_libc::builtin;
diff --git a/libc/src/string/memory_utils/x86_64/memcpy_implementations.h b/libc/src/string/memory_utils/x86_64/memcpy_implementations.h
deleted file mode 100644
index 3844e06e01ab2..0000000000000
--- a/libc/src/string/memory_utils/x86_64/memcpy_implementations.h
+++ /dev/null
@@ -1,84 +0,0 @@
-//===-- Memcpy implementation for x86_64 ------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-#ifndef LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMCPY_IMPLEMENTATIONS_H
-#define LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMCPY_IMPLEMENTATIONS_H
-
-#include "src/__support/macros/config.h" // LIBC_INLINE
-#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
-#include "src/string/memory_utils/op_builtin.h"
-#include "src/string/memory_utils/op_x86.h"
-#include "src/string/memory_utils/utils.h"
-
-#include <stddef.h> // size_t
-
-namespace __llvm_libc {
-
-[[maybe_unused]] LIBC_INLINE void
-inline_memcpy_x86(Ptr __restrict dst, CPtr __restrict src, size_t count) {
- if (count == 0)
- return;
- if (count == 1)
- return builtin::Memcpy<1>::block(dst, src);
- if (count == 2)
- return builtin::Memcpy<2>::block(dst, src);
- if (count == 3)
- return builtin::Memcpy<3>::block(dst, src);
- if (count == 4)
- return builtin::Memcpy<4>::block(dst, src);
- if (count < 8)
- return builtin::Memcpy<4>::head_tail(dst, src, count);
- if (count < 16)
- return builtin::Memcpy<8>::head_tail(dst, src, count);
- if (count < 32)
- return builtin::Memcpy<16>::head_tail(dst, src, count);
- if (count < 64)
- return builtin::Memcpy<32>::head_tail(dst, src, count);
- if (count < 128)
- return builtin::Memcpy<64>::head_tail(dst, src, count);
- if (x86::kAvx && count < 256)
- return builtin::Memcpy<128>::head_tail(dst, src, count);
- builtin::Memcpy<32>::block(dst, src);
- align_to_next_boundary<32, Arg::Dst>(dst, src, count);
- static constexpr size_t kBlockSize = x86::kAvx ? 64 : 32;
- return builtin::Memcpy<kBlockSize>::loop_and_tail(dst, src, count);
-}
-
-[[maybe_unused]] LIBC_INLINE void
-inline_memcpy_x86_maybe_interpose_repmovsb(Ptr __restrict dst,
- CPtr __restrict src, size_t count) {
- // Whether to use rep;movsb exclusively, not at all, or only above a certain
- // threshold.
-#ifndef LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
-#define LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE -1
-#endif
-
-#ifdef LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB
-#error LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB is deprecated use LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE=0 instead.
-#endif // LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB
-
-#ifdef LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
-#error LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE is deprecated use LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE=0 instead.
-#endif // LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
-
- static constexpr size_t kRepMovsbThreshold =
- LIBC_COPT_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE;
- if constexpr (kRepMovsbThreshold == 0) {
- return x86::Memcpy::repmovsb(dst, src, count);
- } else if constexpr (kRepMovsbThreshold == size_t(-1)) {
- return inline_memcpy_x86(dst, src, count);
- } else {
- if (LIBC_UNLIKELY(count >= kRepMovsbThreshold))
- return x86::Memcpy::repmovsb(dst, src, count);
- else
- return inline_memcpy_x86(dst, src, count);
- }
-}
-
-} // namespace __llvm_libc
-
-#endif // LIBC_SRC_STRING_MEMORY_UTILS_X86_64_MEMCPY_IMPLEMENTATIONS_H
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 2f0bcf6f41397..d5ba402e3dc43 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -1404,14 +1404,13 @@ libc_support_library(
"src/string/memory_utils/bzero_implementations.h",
"src/string/memory_utils/memcmp_implementations.h",
"src/string/memory_utils/memcpy_implementations.h",
- "src/string/memory_utils/aarch64/memcpy_implementations.h",
- "src/string/memory_utils/x86_64/memcpy_implementations.h",
"src/string/memory_utils/memmove_implementations.h",
"src/string/memory_utils/memset_implementations.h",
"src/string/memory_utils/strcmp_implementations.h",
"src/string/memory_utils/strstr_implementations.h",
],
deps = [
+ ":__support_common",
":__support_cpp_array",
":__support_cpp_bit",
":__support_cpp_cstddef",
More information about the libc-commits
mailing list