[libc-commits] [libc] 060a43c - [libc][NFC] Move alignment utils to utils.h
Guillaume Chatelet via libc-commits
libc-commits at lists.llvm.org
Thu Sep 29 06:51:49 PDT 2022
Author: Guillaume Chatelet
Date: 2022-09-29T13:51:35Z
New Revision: 060a43ced27ead1ebbe2b0421599f1a03bff25e5
URL: https://github.com/llvm/llvm-project/commit/060a43ced27ead1ebbe2b0421599f1a03bff25e5
DIFF: https://github.com/llvm/llvm-project/commit/060a43ced27ead1ebbe2b0421599f1a03bff25e5.diff
LOG: [libc][NFC] Move alignment utils to utils.h
Added:
Modified:
libc/src/string/memory_utils/elements.h
libc/src/string/memory_utils/utils.h
libc/test/src/string/memory_utils/utils_test.cpp
Removed:
################################################################################
diff --git a/libc/src/string/memory_utils/elements.h b/libc/src/string/memory_utils/elements.h
index a5aa30de97cb..f5a38308d527 100644
--- a/libc/src/string/memory_utils/elements.h
+++ b/libc/src/string/memory_utils/elements.h
@@ -423,8 +423,6 @@ template <typename T, typename TailT = T> struct Loop {
}
};
-enum class Arg { _1, _2, Dst = _1, Src = _2, Lhs = _1, Rhs = _2 };
-
namespace internal {
template <Arg arg> struct ArgSelector {};
@@ -454,10 +452,7 @@ template <Arg arg, size_t Alignment> struct Align {
int additional_bumps = 0) {
auto &aligned_ptr = ArgSelector<arg>::Select(p1ref, p2ref);
auto offset = offset_to_next_aligned<Alignment>(aligned_ptr);
- offset += additional_bumps * Alignment;
- p1ref += offset;
- p2ref += offset;
- size -= offset;
+ adjust(offset + additional_bumps * Alignment, p1ref, p2ref, size);
aligned_ptr = assume_aligned<Alignment>(aligned_ptr);
}
};
diff --git a/libc/src/string/memory_utils/utils.h b/libc/src/string/memory_utils/utils.h
index 7197a7e605e2..d915835e38d8 100644
--- a/libc/src/string/memory_utils/utils.h
+++ b/libc/src/string/memory_utils/utils.h
@@ -28,6 +28,13 @@
namespace __llvm_libc {
+// Allows compile time error reporting in `if constexpr` branches.
+template <bool flag = false>
+static void deferred_static_assert(const char *msg) {
+ static_assert(flag, "compilation error");
+ (void)msg;
+}
+
// Return whether `value` is zero or a power of two.
static constexpr bool is_power2_or_zero(size_t value) {
return (value & (value - 1U)) == 0;
@@ -75,6 +82,83 @@ static inline intptr_t offset_to_next_cache_line(const void *ptr) {
template <size_t alignment, typename T> static T *assume_aligned(T *ptr) {
return reinterpret_cast<T *>(__builtin_assume_aligned(ptr, alignment));
}
+#if defined __has_builtin
+#if __has_builtin(__builtin_memcpy_inline)
+#define LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE
+#endif
+#endif
+
+// Performs a constant count copy.
+template <size_t Size>
+static inline void memcpy_inline(void *__restrict dst,
+ const void *__restrict src) {
+#ifdef LLVM_LIBC_HAS_BUILTIN_MEMCPY_INLINE
+ __builtin_memcpy_inline(dst, src, Size);
+#else
+ for (size_t i = 0; i < Size; ++i)
+ static_cast<char *>(dst)[i] = static_cast<const char *>(src)[i];
+#endif
+}
+
+using Ptr = char *; // Pointer to raw data.
+using CPtr = const char *; // Const pointer to raw data.
+
+// Loads bytes from memory (possibly unaligned) and materializes them as type.
+template <typename T> static inline T load(CPtr ptr) {
+ T Out;
+ memcpy_inline<sizeof(T)>(&Out, ptr);
+ return Out;
+}
+
+// Stores a value of type T in memory (possibly unaligned)
+template <typename T> static inline void store(Ptr ptr, T value) {
+ memcpy_inline<sizeof(T)>(ptr, &value);
+}
+
+// For an operation like memset that operates on a pointer and a count, advances
+// the pointer by offset bytes and decrease count by the same amount.
+static inline void adjust(ptr
diff _t offset, Ptr &ptr, size_t &count) {
+ ptr += offset;
+ count -= offset;
+}
+
+// For an operation like memcpy or memcmp that operates on two pointers and a
+// count, advances the pointers by offset bytes and decrease count by the same
+// amount.
+template <typename T1, typename T2>
+static inline void adjust(ptr
diff _t offset, T1 *__restrict &p1,
+ T2 *__restrict &p2, size_t &count) {
+ p1 += offset;
+ p2 += offset;
+ count -= offset;
+}
+
+// For an operation like memset that operates on a pointer and a count, advances
+// the pointer so it is aligned to SIZE bytes and decrease count by the same
+// amount.
+// We make sure the compiler knows about the adjusted pointer alignment.
+template <size_t SIZE> void align(Ptr &ptr, size_t &count) {
+ adjust(offset_to_next_aligned<SIZE>(ptr), ptr, count);
+ ptr = assume_aligned<SIZE>(ptr);
+}
+
+// For an operation like memcpy or memcmp that operates on two pointers and a
+// count, advances the pointers so one of them gets aligned to SIZE bytes and
+// decrease count by the same amount.
+// We make sure the compiler knows about the adjusted pointer alignment.
+enum class Arg { _1, _2, Dst = _1, Src = _2, Lhs = _1, Rhs = _2 };
+template <size_t SIZE, Arg AlignOn, typename T1, typename T2>
+void align(T1 *__restrict &p1, T2 *__restrict &p2, size_t &count) {
+ if constexpr (AlignOn == Arg::_1) {
+ adjust(offset_to_next_aligned<SIZE>(p1), p1, p2, count);
+ p1 = assume_aligned<SIZE>(p1);
+ } else if constexpr (AlignOn == Arg::_2) {
+ adjust(offset_to_next_aligned<SIZE>(p2), p1, p2, count);
+ p2 = assume_aligned<SIZE>(p2);
+ } else {
+ deferred_static_assert("AlignOn must be either Arg::_1 or Arg::_2");
+ }
+}
} // namespace __llvm_libc
diff --git a/libc/test/src/string/memory_utils/utils_test.cpp b/libc/test/src/string/memory_utils/utils_test.cpp
index f76d3a7fe57e..a20c0900b723 100644
--- a/libc/test/src/string/memory_utils/utils_test.cpp
+++ b/libc/test/src/string/memory_utils/utils_test.cpp
@@ -104,4 +104,71 @@ TEST(LlvmLibcUtilsTest, OffsetToNextCacheLine) {
EXPECT_EQ(offset_to_next_cache_line(forge(LLVM_LIBC_CACHELINE_SIZE - 1)),
I(1));
}
+
+TEST(LlvmLibcUtilsTest, Adjust1) {
+ char a;
+ const size_t base_size = 10;
+ for (size_t I = -2; I < 2; ++I) {
+ auto *ptr = &a;
+ size_t size = base_size;
+ adjust(I, ptr, size);
+ EXPECT_EQ(intptr_t(ptr), intptr_t(&a + I));
+ EXPECT_EQ(size, base_size - I);
+ }
+}
+
+TEST(LlvmLibcUtilsTest, Adjust2) {
+ char a, b;
+ const size_t base_size = 10;
+ for (size_t I = -2; I < 2; ++I) {
+ auto *p1 = &a;
+ auto *p2 = &b;
+ size_t size = base_size;
+ adjust(I, p1, p2, size);
+ EXPECT_EQ(intptr_t(p1), intptr_t(&a + I));
+ EXPECT_EQ(intptr_t(p2), intptr_t(&b + I));
+ EXPECT_EQ(size, base_size - I);
+ }
+}
+
+TEST(LlvmLibcUtilsTest, Align1) {
+ char a;
+ const size_t base_size = 10;
+ {
+ auto *ptr = &a;
+ size_t size = base_size;
+ align<128>(ptr, size);
+ EXPECT_TRUE(uintptr_t(ptr) % 128 == 0);
+ EXPECT_GE(ptr, &a);
+ EXPECT_EQ(size_t(ptr - &a), base_size - size);
+ }
+}
+
+TEST(LlvmLibcUtilsTest, Align2) {
+ char a, b;
+ const size_t base_size = 10;
+ {
+ auto *p1 = &a;
+ auto *p2 = &b;
+ size_t size = base_size;
+ align<128, Arg::_1>(p1, p2, size);
+ EXPECT_TRUE(uintptr_t(p1) % 128 == 0);
+ EXPECT_GE(p1, &a);
+ EXPECT_GE(p2, &b);
+ EXPECT_EQ(size_t(p1 - &a), base_size - size);
+ EXPECT_EQ(size_t(p2 - &b), base_size - size);
+ }
+ {
+ auto *p1 = &a;
+ auto *p2 = &b;
+ size_t size = base_size;
+ align<128, Arg::_2>(p1, p2, size);
+ EXPECT_TRUE(uintptr_t(p2) % 128 == 0);
+ EXPECT_GE(p1, &a);
+ EXPECT_GE(p2, &b);
+ EXPECT_EQ(size_t(p1 - &a), base_size - size);
+ EXPECT_EQ(size_t(p2 - &b), base_size - size);
+ }
+}
+
} // namespace __llvm_libc
More information about the libc-commits
mailing list