[libc-commits] [libc] c2ce8f0 - [libc][NFC] Allow memcpy to be inlined

Guillaume Chatelet via libc-commits libc-commits at lists.llvm.org
Thu Nov 4 06:47:30 PDT 2021


Author: Guillaume Chatelet
Date: 2021-11-04T13:47:19Z
New Revision: c2ce8f02ed948195d0275c3e34e7a6f4a40901e6

URL: https://github.com/llvm/llvm-project/commit/c2ce8f02ed948195d0275c3e34e7a6f4a40901e6
DIFF: https://github.com/llvm/llvm-project/commit/c2ce8f02ed948195d0275c3e34e7a6f4a40901e6.diff

LOG: [libc][NFC] Allow memcpy to be inlined

This allows shipping individual functions without also having to provide
`memcpy` at the expense of bigger functions.
Next is to use this `inlined_memcpy` in:
 - loader/linux/x86_64/start.cpp
 - src/string/memmove.cpp
 - src/string/mempcpy.cpp
 - src/string/strcpy.cpp
 - src/string/strdup.cpp
 - src/string/strndup.cpp

Differential Revision: https://reviews.llvm.org/D113097

Added: 
    libc/src/string/memory_utils/memcpy_implementations.h

Modified: 
    libc/src/string/CMakeLists.txt
    libc/src/string/memcpy.cpp

Removed: 
    libc/src/string/aarch64/memcpy.cpp
    libc/src/string/x86_64/memcpy.cpp


################################################################################
diff  --git a/libc/src/string/CMakeLists.txt b/libc/src/string/CMakeLists.txt
index 0fb50b6c8c353..c7cdcaa913059 100644
--- a/libc/src/string/CMakeLists.txt
+++ b/libc/src/string/CMakeLists.txt
@@ -357,7 +357,7 @@ endif()
 
 function(add_memcpy memcpy_name)
   add_implementation(memcpy ${memcpy_name}
-    SRCS ${MEMCPY_SRC}
+    SRCS ${LIBC_SOURCE_DIR}/src/string/memcpy.cpp
     HDRS ${LIBC_SOURCE_DIR}/src/string/memcpy.h
     DEPENDS
       .memory_utils.memory_utils
@@ -369,7 +369,6 @@ function(add_memcpy memcpy_name)
 endfunction()
 
 if(${LIBC_TARGET_ARCHITECTURE_IS_X86})
-  set(MEMCPY_SRC ${LIBC_SOURCE_DIR}/src/string/x86_64/memcpy.cpp)
   add_memcpy(memcpy_x86_64_opt_sse2   COMPILE_OPTIONS -march=k8             REQUIRE SSE2)
   add_memcpy(memcpy_x86_64_opt_sse4   COMPILE_OPTIONS -march=nehalem        REQUIRE SSE4_2)
   add_memcpy(memcpy_x86_64_opt_avx2   COMPILE_OPTIONS -march=haswell        REQUIRE AVX2)
@@ -377,14 +376,12 @@ if(${LIBC_TARGET_ARCHITECTURE_IS_X86})
   add_memcpy(memcpy_opt_host          COMPILE_OPTIONS ${LIBC_COMPILE_OPTIONS_NATIVE})
   add_memcpy(memcpy)
 elseif(${LIBC_TARGET_ARCHITECTURE_IS_AARCH64})
-  set(MEMCPY_SRC ${LIBC_SOURCE_DIR}/src/string/aarch64/memcpy.cpp)
   # Disable tail merging as it leads to lower performance.
   # Note that '-mllvm' needs to be prefixed with 'SHELL:' to prevent CMake flag deduplication.
   add_memcpy(memcpy_opt_host          COMPILE_OPTIONS ${LIBC_COMPILE_OPTIONS_NATIVE}
                                       COMPILE_OPTIONS "SHELL:-mllvm --tail-merge-threshold=0")
   add_memcpy(memcpy                   COMPILE_OPTIONS "SHELL:-mllvm --tail-merge-threshold=0")
 else()
-  set(MEMCPY_SRC ${LIBC_SOURCE_DIR}/src/string/memcpy.cpp)
   add_memcpy(memcpy_opt_host          COMPILE_OPTIONS ${LIBC_COMPILE_OPTIONS_NATIVE})
   add_memcpy(memcpy)
 endif()

diff  --git a/libc/src/string/aarch64/memcpy.cpp b/libc/src/string/aarch64/memcpy.cpp
deleted file mode 100644
index 1a1fbbc026a73..0000000000000
--- a/libc/src/string/aarch64/memcpy.cpp
+++ /dev/null
@@ -1,77 +0,0 @@
-//===-- Implementation of memcpy ------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/string/memcpy.h"
-#include "src/__support/common.h"
-#include "src/string/memory_utils/elements.h"
-
-namespace __llvm_libc {
-
-using _1 = scalar::UINT8;
-using _2 = scalar::UINT16;
-using _3 = Chained<scalar::UINT16, scalar::UINT8>;
-using _4 = scalar::UINT32;
-using _8 = scalar::UINT64;
-using _16 = Repeated<scalar::UINT64, 2>;
-using _32 = Repeated<scalar::UINT64, 4>;
-using _64 = Repeated<scalar::UINT64, 8>;
-
-// Design rationale
-// ================
-//
-// Using a profiler to observe size distributions for calls into libc
-// functions, it was found most operations act on a small number of bytes.
-// This makes it important to favor small sizes.
-//
-// We have used __builtin_expect to tell the compiler to favour lower sizes as
-// that will reduce the branching overhead where that would hurt most
-// proportional to total cost of copying.
-//
-// The function is written in C++ for several reasons:
-// - The compiler can __see__ the code, this is useful when performing Profile
-//   Guided Optimization as the optimized code can take advantage of branching
-//   probabilities.
-// - It also allows for easier customization and favors testing multiple
-//   implementation parameters.
-// - As compilers and processors get better, the generated code is improved
-//   with little change on the code side.
-// This implementation has been tuned for Neoverse-N1.
-static void memcpy_aarch64(char *__restrict dst, const char *__restrict src,
-                           size_t count) {
-  if (count == 0)
-    return;
-  if (count == 1)
-    return Copy<_1>(dst, src);
-  if (count == 2)
-    return Copy<_2>(dst, src);
-  if (count == 3)
-    return Copy<_3>(dst, src);
-  if (count == 4)
-    return Copy<_4>(dst, src);
-  if (count < 8)
-    return Copy<HeadTail<_4>>(dst, src, count);
-  if (count < 16)
-    return Copy<HeadTail<_8>>(dst, src, count);
-  if (count < 32)
-    return Copy<HeadTail<_16>>(dst, src, count);
-  if (count < 64)
-    return Copy<HeadTail<_32>>(dst, src, count);
-  if (count < 128)
-    return Copy<HeadTail<_64>>(dst, src, count);
-  return Copy<Align<_16, Arg::Src>::Then<Loop<_64>>>(dst, src, count);
-}
-
-LLVM_LIBC_FUNCTION(void *, memcpy,
-                   (void *__restrict dst, const void *__restrict src,
-                    size_t size)) {
-  memcpy_aarch64(reinterpret_cast<char *>(dst),
-                 reinterpret_cast<const char *>(src), size);
-  return dst;
-}
-
-} // namespace __llvm_libc

diff  --git a/libc/src/string/memcpy.cpp b/libc/src/string/memcpy.cpp
index 5e70e00db1b91..ff990f48a20bc 100644
--- a/libc/src/string/memcpy.cpp
+++ b/libc/src/string/memcpy.cpp
@@ -8,61 +8,15 @@
 
 #include "src/string/memcpy.h"
 #include "src/__support/common.h"
-#include "src/string/memory_utils/elements.h"
+#include "src/string/memory_utils/memcpy_implementations.h"
 
 namespace __llvm_libc {
 
-// Design rationale
-// ================
-//
-// Using a profiler to observe size distributions for calls into libc
-// functions, it was found most operations act on a small number of bytes.
-// This makes it important to favor small sizes.
-//
-// The tests for `count` are in ascending order so the cost of branching is
-// proportional to the cost of copying.
-//
-// The function is written in C++ for several reasons:
-// - The compiler can __see__ the code, this is useful when performing Profile
-//   Guided Optimization as the optimized code can take advantage of branching
-//   probabilities.
-// - It also allows for easier customization and favors testing multiple
-//   implementation parameters.
-// - As compilers and processors get better, the generated code is improved
-//   with little change on the code side.
-static void memcpy_impl(char *__restrict dst, const char *__restrict src,
-                        size_t count) {
-  // Use scalar strategies (_1, _2, _3 ...)
-  using namespace __llvm_libc::scalar;
-
-  if (count == 0)
-    return;
-  if (count == 1)
-    return Copy<_1>(dst, src);
-  if (count == 2)
-    return Copy<_2>(dst, src);
-  if (count == 3)
-    return Copy<_3>(dst, src);
-  if (count == 4)
-    return Copy<_4>(dst, src);
-  if (count < 8)
-    return Copy<HeadTail<_4>>(dst, src, count);
-  if (count < 16)
-    return Copy<HeadTail<_8>>(dst, src, count);
-  if (count < 32)
-    return Copy<HeadTail<_16>>(dst, src, count);
-  if (count < 64)
-    return Copy<HeadTail<_32>>(dst, src, count);
-  if (count < 128)
-    return Copy<HeadTail<_64>>(dst, src, count);
-  return Copy<Align<_32, Arg::Src>::Then<Loop<_32>>>(dst, src, count);
-}
-
 LLVM_LIBC_FUNCTION(void *, memcpy,
                    (void *__restrict dst, const void *__restrict src,
                     size_t size)) {
-  memcpy_impl(reinterpret_cast<char *>(dst),
-              reinterpret_cast<const char *>(src), size);
+  inline_memcpy(reinterpret_cast<char *>(dst),
+                reinterpret_cast<const char *>(src), size);
   return dst;
 }
 

diff  --git a/libc/src/string/memory_utils/memcpy_implementations.h b/libc/src/string/memory_utils/memcpy_implementations.h
new file mode 100644
index 0000000000000..2a738f7ecf1d3
--- /dev/null
+++ b/libc/src/string/memory_utils/memcpy_implementations.h
@@ -0,0 +1,157 @@
+//===-- Memcpy implementation -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H
+#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H
+
+#include "src/__support/architectures.h"
+#include "src/__support/common.h"
+#include "src/string/memory_utils/elements.h"
+#include "src/string/memory_utils/utils.h"
+
+#include <stddef.h> // size_t
+
+// Design rationale
+// ================
+//
+// Using a profiler to observe size distributions for calls into libc
+// functions, it was found most operations act on a small number of bytes.
+// This makes it important to favor small sizes.
+//
+// The tests for `count` are in ascending order so the cost of branching is
+// proportional to the cost of copying.
+//
+// The function is written in C++ for several reasons:
+// - The compiler can __see__ the code, this is useful when performing Profile
+//   Guided Optimization as the optimized code can take advantage of branching
+//   probabilities.
+// - It also allows for easier customization and favors testing multiple
+//   implementation parameters.
+// - As compilers and processors get better, the generated code is improved
+//   with little change on the code side.
+
+namespace __llvm_libc {
+
+static inline void inline_memcpy(char *__restrict dst,
+                                 const char *__restrict src, size_t count) {
+#if defined(LLVM_LIBC_ARCH_X86)
+  /////////////////////////////////////////////////////////////////////////////
+  // LLVM_LIBC_ARCH_X86
+  /////////////////////////////////////////////////////////////////////////////
+  using namespace __llvm_libc::x86;
+
+  // Whether to use only rep;movsb.
+  constexpr bool kUseOnlyRepMovsb =
+      LLVM_LIBC_IS_DEFINED(LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB);
+
+  // kRepMovsBSize == -1 : Only CopyAligned is used.
+  // kRepMovsBSize ==  0 : Only RepMovsb is used.
+  // else CopyAligned is used up to kRepMovsBSize and then RepMovsb.
+  constexpr size_t kRepMovsBSize =
+#if defined(LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE)
+      LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE;
+#else
+      -1;
+#endif // LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
+
+  // Whether target supports AVX instructions.
+  constexpr bool kHasAvx = LLVM_LIBC_IS_DEFINED(__AVX__);
+
+#if defined(__AVX__)
+  using LoopBlockSize = _64;
+#else
+  using LoopBlockSize = _32;
+#endif
+
+  if (kUseOnlyRepMovsb)
+    return Copy<Accelerator>(dst, src, count);
+
+  if (count == 0)
+    return;
+  if (count == 1)
+    return Copy<_1>(dst, src);
+  if (count == 2)
+    return Copy<_2>(dst, src);
+  if (count == 3)
+    return Copy<_3>(dst, src);
+  if (count == 4)
+    return Copy<_4>(dst, src);
+  if (count < 8)
+    return Copy<HeadTail<_4>>(dst, src, count);
+  if (count < 16)
+    return Copy<HeadTail<_8>>(dst, src, count);
+  if (count < 32)
+    return Copy<HeadTail<_16>>(dst, src, count);
+  if (count < 64)
+    return Copy<HeadTail<_32>>(dst, src, count);
+  if (count < 128)
+    return Copy<HeadTail<_64>>(dst, src, count);
+  if (kHasAvx && count < 256)
+    return Copy<HeadTail<_128>>(dst, src, count);
+  if (count <= kRepMovsBSize)
+    return Copy<Align<_32, Arg::Dst>::Then<Loop<LoopBlockSize>>>(dst, src,
+                                                                 count);
+  return Copy<Accelerator>(dst, src, count);
+#elif defined(LLVM_LIBC_ARCH_AARCH64)
+  /////////////////////////////////////////////////////////////////////////////
+  // LLVM_LIBC_ARCH_AARCH64
+  /////////////////////////////////////////////////////////////////////////////
+  using namespace __llvm_libc::scalar;
+  if (count == 0)
+    return;
+  if (count == 1)
+    return Copy<_1>(dst, src);
+  if (count == 2)
+    return Copy<_2>(dst, src);
+  if (count == 3)
+    return Copy<_3>(dst, src);
+  if (count == 4)
+    return Copy<_4>(dst, src);
+  if (count < 8)
+    return Copy<HeadTail<_4>>(dst, src, count);
+  if (count < 16)
+    return Copy<HeadTail<_8>>(dst, src, count);
+  if (count < 32)
+    return Copy<HeadTail<_16>>(dst, src, count);
+  if (count < 64)
+    return Copy<HeadTail<_32>>(dst, src, count);
+  if (count < 128)
+    return Copy<HeadTail<_64>>(dst, src, count);
+  return Copy<Align<_16, Arg::Src>::Then<Loop<_64>>>(dst, src, count);
+#else
+  /////////////////////////////////////////////////////////////////////////////
+  // Default
+  /////////////////////////////////////////////////////////////////////////////
+  using namespace __llvm_libc::scalar;
+  if (count == 0)
+    return;
+  if (count == 1)
+    return Copy<_1>(dst, src);
+  if (count == 2)
+    return Copy<_2>(dst, src);
+  if (count == 3)
+    return Copy<_3>(dst, src);
+  if (count == 4)
+    return Copy<_4>(dst, src);
+  if (count < 8)
+    return Copy<HeadTail<_4>>(dst, src, count);
+  if (count < 16)
+    return Copy<HeadTail<_8>>(dst, src, count);
+  if (count < 32)
+    return Copy<HeadTail<_16>>(dst, src, count);
+  if (count < 64)
+    return Copy<HeadTail<_32>>(dst, src, count);
+  if (count < 128)
+    return Copy<HeadTail<_64>>(dst, src, count);
+  return Copy<Align<_32, Arg::Src>::Then<Loop<_32>>>(dst, src, count);
+#endif
+}
+
+} // namespace __llvm_libc
+
+#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY_IMPLEMENTATIONS_H

diff  --git a/libc/src/string/x86_64/memcpy.cpp b/libc/src/string/x86_64/memcpy.cpp
deleted file mode 100644
index 7f6e5b64b3a74..0000000000000
--- a/libc/src/string/x86_64/memcpy.cpp
+++ /dev/null
@@ -1,109 +0,0 @@
-//===-- Implementation of memcpy ------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "src/string/memcpy.h"
-#include "src/__support/common.h"
-#include "src/string/memory_utils/elements.h"
-
-namespace __llvm_libc {
-
-// Whether to use only rep;movsb.
-constexpr bool kUseOnlyRepMovsb =
-    LLVM_LIBC_IS_DEFINED(LLVM_LIBC_MEMCPY_X86_USE_ONLY_REPMOVSB);
-
-// kRepMovsBSize == -1 : Only CopyAligned is used.
-// kRepMovsBSize ==  0 : Only RepMovsb is used.
-// else CopyAligned is used up to kRepMovsBSize and then RepMovsb.
-constexpr size_t kRepMovsBSize =
-#ifdef LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
-    LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE;
-#else
-    -1;
-#endif // LLVM_LIBC_MEMCPY_X86_USE_REPMOVSB_FROM_SIZE
-
-// Whether target supports AVX instructions.
-constexpr bool kHasAvx = LLVM_LIBC_IS_DEFINED(__AVX__);
-
-#ifdef __AVX__
-using LoopBlockSize = __llvm_libc::x86::_64;
-#else
-using LoopBlockSize = __llvm_libc::x86::_32;
-#endif
-
-static void CopyRepMovsb(char *__restrict dst, const char *__restrict src,
-                         size_t count) {
-  // FIXME: Add MSVC support with
-  // #include <intrin.h>
-  // __movsb(reinterpret_cast<unsigned char *>(dst),
-  //         reinterpret_cast<const unsigned char *>(src), count);
-  asm volatile("rep movsb" : "+D"(dst), "+S"(src), "+c"(count) : : "memory");
-}
-
-// Design rationale
-// ================
-//
-// Using a profiler to observe size distributions for calls into libc
-// functions, it was found most operations act on a small number of bytes.
-// This makes it important to favor small sizes.
-//
-// The tests for `count` are in ascending order so the cost of branching is
-// proportional to the cost of copying.
-//
-// The function is written in C++ for several reasons:
-// - The compiler can __see__ the code, this is useful when performing Profile
-//   Guided Optimization as the optimized code can take advantage of branching
-//   probabilities.
-// - It also allows for easier customization and favors testing multiple
-//   implementation parameters.
-// - As compilers and processors get better, the generated code is improved
-//   with little change on the code side.
-static void memcpy_x86(char *__restrict dst, const char *__restrict src,
-                       size_t count) {
-  // Use x86 strategies (_1, _2, _3 ...)
-  using namespace __llvm_libc::x86;
-
-  if (kUseOnlyRepMovsb)
-    return CopyRepMovsb(dst, src, count);
-
-  if (count == 0)
-    return;
-  if (count == 1)
-    return Copy<_1>(dst, src);
-  if (count == 2)
-    return Copy<_2>(dst, src);
-  if (count == 3)
-    return Copy<_3>(dst, src);
-  if (count == 4)
-    return Copy<_4>(dst, src);
-  if (count < 8)
-    return Copy<HeadTail<_4>>(dst, src, count);
-  if (count < 16)
-    return Copy<HeadTail<_8>>(dst, src, count);
-  if (count < 32)
-    return Copy<HeadTail<_16>>(dst, src, count);
-  if (count < 64)
-    return Copy<HeadTail<_32>>(dst, src, count);
-  if (count < 128)
-    return Copy<HeadTail<_64>>(dst, src, count);
-  if (kHasAvx && count < 256)
-    return Copy<HeadTail<_128>>(dst, src, count);
-  if (count <= kRepMovsBSize)
-    return Copy<Align<_32, Arg::Dst>::Then<Loop<LoopBlockSize>>>(dst, src,
-                                                                 count);
-  return CopyRepMovsb(dst, src, count);
-}
-
-LLVM_LIBC_FUNCTION(void *, memcpy,
-                   (void *__restrict dst, const void *__restrict src,
-                    size_t size)) {
-  memcpy_x86(reinterpret_cast<char *>(dst), reinterpret_cast<const char *>(src),
-             size);
-  return dst;
-}
-
-} // namespace __llvm_libc


        


More information about the libc-commits mailing list