[compiler-rt] [compiler-rt][AArch64] Provide basic implementations of SME memcpy/memmove in case of strictly aligned memory access (PR #138250)
Victor Campos via llvm-commits
llvm-commits at lists.llvm.org
Fri May 30 05:26:46 PDT 2025
https://github.com/vhscampos updated https://github.com/llvm/llvm-project/pull/138250
>From cd24ec57a84ed837e3e22d589de15e3ed3691ddb Mon Sep 17 00:00:00 2001
From: Victor Campos <victor.campos at arm.com>
Date: Thu, 1 May 2025 13:31:36 +0100
Subject: [PATCH 1/2] [compiler-rt][AArch64] Provide basic implementations of
SME memcpy/memmove in case of strictly aligned memory access
The existing implementations, written in assembly, make use of unaligned
accesses for performance reasons. They are not compatible with strict
aligned configurations, i.e. with `-mno-unaligned-access`.
If the functions are used in this scenario, an exception is raised due
to unaligned memory accesses.
This patch reintroduces vanilla implementations for these functions to
be used in strictly aligned configurations. The actual code is largely
based on the code from https://github.com/llvm/llvm-project/pull/77496
---
.../builtins/aarch64/sme-libc-mem-routines.S | 6 ++-
.../lib/builtins/aarch64/sme-libc-routines.c | 48 ++++++++++++++++++-
2 files changed, 51 insertions(+), 3 deletions(-)
diff --git a/compiler-rt/lib/builtins/aarch64/sme-libc-mem-routines.S b/compiler-rt/lib/builtins/aarch64/sme-libc-mem-routines.S
index e736829967c0c..cba4a2cbc4fa9 100644
--- a/compiler-rt/lib/builtins/aarch64/sme-libc-mem-routines.S
+++ b/compiler-rt/lib/builtins/aarch64/sme-libc-mem-routines.S
@@ -6,6 +6,8 @@
#include "../assembly.h"
+#ifdef __ARM_FEATURE_UNALIGNED
+
//
// __arm_sc_memcpy / __arm_sc_memmove
//
@@ -346,4 +348,6 @@ DEFINE_COMPILERRT_FUNCTION(__arm_sc_memset)
ret
END_COMPILERRT_FUNCTION(__arm_sc_memset)
-#endif // __aarch64__
+#endif /* defined(__aarch64__) && __ARM_FP != 0 */
+
+#endif /* __ARM_FEATURE_UNALIGNED */
diff --git a/compiler-rt/lib/builtins/aarch64/sme-libc-routines.c b/compiler-rt/lib/builtins/aarch64/sme-libc-routines.c
index 07d6681485556..24b9b3bcb49da 100644
--- a/compiler-rt/lib/builtins/aarch64/sme-libc-routines.c
+++ b/compiler-rt/lib/builtins/aarch64/sme-libc-routines.c
@@ -1,7 +1,8 @@
#include <stddef.h>
-/* The asm version uses FP registers. Use this on targets without them */
-#if __ARM_FP == 0
+// The asm version uses FP registers and unaligned memory accesses. Use this on
+// targets without them.
+#if __ARM_FP == 0 || !defined(__ARM_FEATURE_UNALIGNED)
void *__arm_sc_memset(void *dest, int c, size_t n) __arm_streaming_compatible {
unsigned char *destp = (unsigned char *)dest;
unsigned char c8 = (unsigned char)c;
@@ -22,3 +23,46 @@ const void *__arm_sc_memchr(const void *src, int c,
return NULL;
}
+
+#ifndef __ARM_FEATURE_UNALIGNED
+
+static void *memcpy_fwd(void *dest, const void *src,
+ size_t n) __arm_streaming_compatible {
+ unsigned char *destp = (unsigned char *)dest;
+ const unsigned char *srcp = (const unsigned char *)src;
+
+ for (size_t i = 0; i < n; ++i)
+ destp[i] = srcp[i];
+ return dest;
+}
+
+static void *memcpy_rev(void *dest, const void *src,
+ size_t n) __arm_streaming_compatible {
+ unsigned char *destp = (unsigned char *)dest;
+ const unsigned char *srcp = (const unsigned char *)src;
+
+ while (n > 0) {
+ --n;
+ destp[n] = srcp[n];
+ }
+ return dest;
+}
+
+void *__arm_sc_memcpy(void *__restrict dest, const void *__restrict src,
+ size_t n) __arm_streaming_compatible {
+ return memcpy_fwd(dest, src, n);
+}
+
+void *__arm_sc_memmove(void *dest, const void *src,
+ size_t n) __arm_streaming_compatible {
+ unsigned char *destp = (unsigned char *)dest;
+ const unsigned char *srcp = (const unsigned char *)src;
+
+ if ((srcp > (destp + n)) || (destp > (srcp + n)))
+ return __arm_sc_memcpy(dest, src, n);
+ if (srcp > destp)
+ return memcpy_fwd(dest, src, n);
+ return memcpy_rev(dest, src, n);
+}
+
+#endif /* !defined(__ARM_FEATURE_UNALIGNED) */
>From ca036e8d9befeec7a4bf4e1458ceb00c464f48d7 Mon Sep 17 00:00:00 2001
From: Victor Campos <victor.campos at arm.com>
Date: Fri, 30 May 2025 11:12:29 +0100
Subject: [PATCH 2/2] Changes: - Split functions into separate files. - Select
which implementation to use based on target features. The selection is now
done in CMake.
---
compiler-rt/cmake/builtin-config-ix.cmake | 18 ++
compiler-rt/lib/builtins/CMakeLists.txt | 12 +-
.../aarch64/sme-libc-memcpy-memmove.c | 55 ++++
.../builtins/aarch64/sme-libc-memset-memchr.c | 37 +++
...utines.S => sme-libc-opt-memcpy-memmove.S} | 133 +--------
.../aarch64/sme-libc-opt-memset-memchr.S | 261 ++++++++++++++++++
.../lib/builtins/aarch64/sme-libc-routines.c | 68 -----
7 files changed, 395 insertions(+), 189 deletions(-)
create mode 100644 compiler-rt/lib/builtins/aarch64/sme-libc-memcpy-memmove.c
create mode 100644 compiler-rt/lib/builtins/aarch64/sme-libc-memset-memchr.c
rename compiler-rt/lib/builtins/aarch64/{sme-libc-mem-routines.S => sme-libc-opt-memcpy-memmove.S} (69%)
create mode 100644 compiler-rt/lib/builtins/aarch64/sme-libc-opt-memset-memchr.S
delete mode 100644 compiler-rt/lib/builtins/aarch64/sme-libc-routines.c
diff --git a/compiler-rt/cmake/builtin-config-ix.cmake b/compiler-rt/cmake/builtin-config-ix.cmake
index cbb43a5958d2f..b6aea17ec10f2 100644
--- a/compiler-rt/cmake/builtin-config-ix.cmake
+++ b/compiler-rt/cmake/builtin-config-ix.cmake
@@ -50,6 +50,24 @@ void foo(void) __arm_streaming_compatible {
}
")
+builtin_check_c_compiler_source(COMPILER_RT_HAS_ARM_UNALIGNED
+"
+void foo() {
+#ifndef __ARM_FEATURE_UNALIGNED
+#error \"Unaligned accesses unsupported\"
+#endif
+}
+")
+
+builtin_check_c_compiler_source(COMPILER_RT_HAS_ARM_FP
+"
+void foo() {
+#ifndef __ARM_FP
+#error \"No floating-point support\"
+#endif
+}
+")
+
check_include_files("sys/auxv.h" COMPILER_RT_HAS_AUXV)
if(ANDROID)
diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt
index 0c986f484bf5e..634a817674f7b 100644
--- a/compiler-rt/lib/builtins/CMakeLists.txt
+++ b/compiler-rt/lib/builtins/CMakeLists.txt
@@ -582,9 +582,17 @@ if (COMPILER_RT_HAS_AARCH64_SME)
set_source_files_properties(aarch64/arm_apple_sme_abi.s PROPERTIES COMPILE_FLAGS -march=armv8a+sme)
message(STATUS "AArch64 Apple SME ABI routines enabled")
elseif (NOT COMPILER_RT_DISABLE_AARCH64_FMV AND COMPILER_RT_HAS_FNO_BUILTIN_FLAG AND COMPILER_RT_AARCH64_FMV_USES_GLOBAL_CONSTRUCTOR)
- list(APPEND aarch64_SOURCES aarch64/sme-abi.S aarch64/sme-libc-mem-routines.S aarch64/sme-abi-assert.c aarch64/sme-libc-routines.c)
+ if(COMPILER_RT_HAS_ARM_UNALIGNED AND COMPILER_RT_HAS_ARM_FP)
+ list(APPEND aarch64_SOURCES aarch64/sme-libc-opt-memset-memchr.S aarch64/sme-libc-opt-memcpy-memmove.S)
+ elseif(COMPILER_RT_HAS_ARM_UNALIGNED)
+ list(APPEND aarch64_SOURCES aarch64/sme-libc-memset-memchr.c aarch64/sme-libc-opt-memcpy-memmove.S)
+ message(WARNING "AArch64 SME ABI assembly-optimized memset/memchr disabled: target does not have hardware floating-point support.")
+ else()
+ list(APPEND aarch64_SOURCES aarch64/sme-libc-memset-memchr.c aarch64/sme-libc-memcpy-memmove.c)
+ message(WARNING "AArch64 SME ABI assembly-optimized routines disabled: target does not support unaligned accesses.")
+ endif()
message(STATUS "AArch64 SME ABI routines enabled")
- set_source_files_properties(aarch64/sme-libc-routines.c PROPERTIES COMPILE_FLAGS "-fno-builtin")
+ set_source_files_properties(aarch64/sme-libc-memset-memchr.c aarch64/sme-libc-memcpy-memmove.c PROPERTIES COMPILE_FLAGS "-fno-builtin")
else()
if(COMPILER_RT_DISABLE_AARCH64_FMV)
message(WARNING "AArch64 SME ABI routines require function multiversioning support.")
diff --git a/compiler-rt/lib/builtins/aarch64/sme-libc-memcpy-memmove.c b/compiler-rt/lib/builtins/aarch64/sme-libc-memcpy-memmove.c
new file mode 100644
index 0000000000000..9792079964d55
--- /dev/null
+++ b/compiler-rt/lib/builtins/aarch64/sme-libc-memcpy-memmove.c
@@ -0,0 +1,55 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains basic implementations of Scalable Matrix Extension (SME)
+/// compatible memcpy and memmove functions to be used when their assembly-
+/// optimized counterparts can't.
+///
+//===----------------------------------------------------------------------===//
+
+#include <stddef.h>
+
+static void *__arm_sc_memcpy_fwd(void *dest, const void *src,
+ size_t n) __arm_streaming_compatible {
+ unsigned char *destp = (unsigned char *)dest;
+ const unsigned char *srcp = (const unsigned char *)src;
+
+ for (size_t i = 0; i < n; ++i)
+ destp[i] = srcp[i];
+ return dest;
+}
+
+static void *__arm_sc_memcpy_rev(void *dest, const void *src,
+ size_t n) __arm_streaming_compatible {
+ unsigned char *destp = (unsigned char *)dest;
+ const unsigned char *srcp = (const unsigned char *)src;
+
+ while (n > 0) {
+ --n;
+ destp[n] = srcp[n];
+ }
+ return dest;
+}
+
+extern void *__arm_sc_memcpy(void *__restrict dest, const void *__restrict src,
+ size_t n) __arm_streaming_compatible {
+ return __arm_sc_memcpy_fwd(dest, src, n);
+}
+
+extern void *__arm_sc_memmove(void *dest, const void *src,
+ size_t n) __arm_streaming_compatible {
+ unsigned char *destp = (unsigned char *)dest;
+ const unsigned char *srcp = (const unsigned char *)src;
+
+ if ((srcp > (destp + n)) || (destp > (srcp + n)))
+ return __arm_sc_memcpy(dest, src, n);
+ if (srcp > destp)
+ return __arm_sc_memcpy_fwd(dest, src, n);
+ return __arm_sc_memcpy_rev(dest, src, n);
+}
diff --git a/compiler-rt/lib/builtins/aarch64/sme-libc-memset-memchr.c b/compiler-rt/lib/builtins/aarch64/sme-libc-memset-memchr.c
new file mode 100644
index 0000000000000..85873c3554fbd
--- /dev/null
+++ b/compiler-rt/lib/builtins/aarch64/sme-libc-memset-memchr.c
@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains basic implementations of Scalable Matrix Extension (SME)
+/// compatible memset and memchr functions to be used when their assembly-
+/// optimized counterparts can't.
+///
+//===----------------------------------------------------------------------===//
+
+#include <stddef.h>
+
+extern void *__arm_sc_memset(void *dest, int c,
+ size_t n) __arm_streaming_compatible {
+ unsigned char *destp = (unsigned char *)dest;
+ unsigned char c8 = (unsigned char)c;
+ for (size_t i = 0; i < n; ++i)
+ destp[i] = c8;
+
+ return dest;
+}
+
+extern const void *__arm_sc_memchr(const void *src, int c,
+ size_t n) __arm_streaming_compatible {
+ const unsigned char *srcp = (const unsigned char *)src;
+ unsigned char c8 = (unsigned char)c;
+ for (size_t i = 0; i < n; ++i)
+ if (srcp[i] == c8)
+ return &srcp[i];
+
+ return NULL;
+}
\ No newline at end of file
diff --git a/compiler-rt/lib/builtins/aarch64/sme-libc-mem-routines.S b/compiler-rt/lib/builtins/aarch64/sme-libc-opt-memcpy-memmove.S
similarity index 69%
rename from compiler-rt/lib/builtins/aarch64/sme-libc-mem-routines.S
rename to compiler-rt/lib/builtins/aarch64/sme-libc-opt-memcpy-memmove.S
index cba4a2cbc4fa9..8bc759a29a312 100644
--- a/compiler-rt/lib/builtins/aarch64/sme-libc-mem-routines.S
+++ b/compiler-rt/lib/builtins/aarch64/sme-libc-opt-memcpy-memmove.S
@@ -1,13 +1,23 @@
+//===----------------------------------------------------------------------===//
+//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-// Routines taken from libc/AOR_v20.02/string/aarch64
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains assembly-optimized implementations of Scalable Matrix
+/// Extension (SME) compatible memcpy and memmove functions.
+///
+/// These implementations depend on unaligned access support.
+///
+/// Routines taken from libc/AOR_v20.02/string/aarch64.
+///
+//===----------------------------------------------------------------------===//
#include "../assembly.h"
-#ifdef __ARM_FEATURE_UNALIGNED
-
//
// __arm_sc_memcpy / __arm_sc_memmove
//
@@ -236,118 +246,3 @@ END_COMPILERRT_FUNCTION(__arm_sc_memcpy)
DEFINE_COMPILERRT_FUNCTION_ALIAS(__arm_sc_memmove, __arm_sc_memcpy)
-// This version uses FP registers. Use this only on targets with them
-#if defined(__aarch64__) && __ARM_FP != 0
-//
-// __arm_sc_memset
-//
-
-#define dstin x0
-#define val x1
-#define valw w1
-#define count x2
-#define dst x3
-#define dstend2 x4
-#define zva_val x5
-
-DEFINE_COMPILERRT_FUNCTION(__arm_sc_memset)
-#ifdef __ARM_FEATURE_SVE
- mov z0.b, valw
-#else
- bfi valw, valw, #8, #8
- bfi valw, valw, #16, #16
- bfi val, val, #32, #32
- fmov d0, val
- fmov v0.d[1], val
-#endif
- add dstend2, dstin, count
-
- cmp count, 96
- b.hi 7f // set_long
- cmp count, 16
- b.hs 4f // set_medium
- mov val, v0.D[0]
-
- /* Set 0..15 bytes. */
- tbz count, 3, 1f
- str val, [dstin]
- str val, [dstend2, -8]
- ret
- nop
-1: tbz count, 2, 2f
- str valw, [dstin]
- str valw, [dstend2, -4]
- ret
-2: cbz count, 3f
- strb valw, [dstin]
- tbz count, 1, 3f
- strh valw, [dstend2, -2]
-3: ret
-
- /* Set 17..96 bytes. */
-4: // set_medium
- str q0, [dstin]
- tbnz count, 6, 6f // set96
- str q0, [dstend2, -16]
- tbz count, 5, 5f
- str q0, [dstin, 16]
- str q0, [dstend2, -32]
-5: ret
-
- .p2align 4
- /* Set 64..96 bytes. Write 64 bytes from the start and
- 32 bytes from the end. */
-6: // set96
- str q0, [dstin, 16]
- stp q0, q0, [dstin, 32]
- stp q0, q0, [dstend2, -32]
- ret
-
- .p2align 4
-7: // set_long
- and valw, valw, 255
- bic dst, dstin, 15
- str q0, [dstin]
- cmp count, 160
- ccmp valw, 0, 0, hs
- b.ne 9f // no_zva
-
-#ifndef SKIP_ZVA_CHECK
- mrs zva_val, dczid_el0
- and zva_val, zva_val, 31
- cmp zva_val, 4 /* ZVA size is 64 bytes. */
- b.ne 9f // no_zva
-#endif
- str q0, [dst, 16]
- stp q0, q0, [dst, 32]
- bic dst, dst, 63
- sub count, dstend2, dst /* Count is now 64 too large. */
- sub count, count, 128 /* Adjust count and bias for loop. */
-
- .p2align 4
-8: // zva_loop
- add dst, dst, 64
- dc zva, dst
- subs count, count, 64
- b.hi 8b // zva_loop
- stp q0, q0, [dstend2, -64]
- stp q0, q0, [dstend2, -32]
- ret
-
-9: // no_zva
- sub count, dstend2, dst /* Count is 16 too large. */
- sub dst, dst, 16 /* Dst is biased by -32. */
- sub count, count, 64 + 16 /* Adjust count and bias for loop. */
-10: // no_zva_loop
- stp q0, q0, [dst, 32]
- stp q0, q0, [dst, 64]!
- subs count, count, 64
- b.hi 10b // no_zva_loop
- stp q0, q0, [dstend2, -64]
- stp q0, q0, [dstend2, -32]
- ret
-END_COMPILERRT_FUNCTION(__arm_sc_memset)
-
-#endif /* defined(__aarch64__) && __ARM_FP != 0 */
-
-#endif /* __ARM_FEATURE_UNALIGNED */
diff --git a/compiler-rt/lib/builtins/aarch64/sme-libc-opt-memset-memchr.S b/compiler-rt/lib/builtins/aarch64/sme-libc-opt-memset-memchr.S
new file mode 100644
index 0000000000000..b6939f114cbae
--- /dev/null
+++ b/compiler-rt/lib/builtins/aarch64/sme-libc-opt-memset-memchr.S
@@ -0,0 +1,261 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains assembly-optimized implementations of Scalable Matrix
+/// Extension (SME) compatible memset and memchr functions.
+///
+/// These implementations depend on unaligned access and floating-point support.
+///
+/// Routines taken from libc/AOR_v20.02/string/aarch64.
+///
+//===----------------------------------------------------------------------===//
+
+#include "../assembly.h"
+
+//
+// __arm_sc_memset
+//
+
+#define dstin x0
+#define val x1
+#define valw w1
+#define count x2
+#define dst x3
+#define dstend2 x4
+#define zva_val x5
+
+DEFINE_COMPILERRT_FUNCTION(__arm_sc_memset)
+#ifdef __ARM_FEATURE_SVE
+ mov z0.b, valw
+#else
+ bfi valw, valw, #8, #8
+ bfi valw, valw, #16, #16
+ bfi val, val, #32, #32
+ fmov d0, val
+ fmov v0.d[1], val
+#endif
+ add dstend2, dstin, count
+
+ cmp count, 96
+ b.hi 7f // set_long
+ cmp count, 16
+ b.hs 4f // set_medium
+ mov val, v0.D[0]
+
+ /* Set 0..15 bytes. */
+ tbz count, 3, 1f
+ str val, [dstin]
+ str val, [dstend2, -8]
+ ret
+ nop
+1: tbz count, 2, 2f
+ str valw, [dstin]
+ str valw, [dstend2, -4]
+ ret
+2: cbz count, 3f
+ strb valw, [dstin]
+ tbz count, 1, 3f
+ strh valw, [dstend2, -2]
+3: ret
+
+ /* Set 17..96 bytes. */
+4: // set_medium
+ str q0, [dstin]
+ tbnz count, 6, 6f // set96
+ str q0, [dstend2, -16]
+ tbz count, 5, 5f
+ str q0, [dstin, 16]
+ str q0, [dstend2, -32]
+5: ret
+
+ .p2align 4
+ /* Set 64..96 bytes. Write 64 bytes from the start and
+ 32 bytes from the end. */
+6: // set96
+ str q0, [dstin, 16]
+ stp q0, q0, [dstin, 32]
+ stp q0, q0, [dstend2, -32]
+ ret
+
+ .p2align 4
+7: // set_long
+ and valw, valw, 255
+ bic dst, dstin, 15
+ str q0, [dstin]
+ cmp count, 160
+ ccmp valw, 0, 0, hs
+ b.ne 9f // no_zva
+
+#ifndef SKIP_ZVA_CHECK
+ mrs zva_val, dczid_el0
+ and zva_val, zva_val, 31
+ cmp zva_val, 4 /* ZVA size is 64 bytes. */
+ b.ne 9f // no_zva
+#endif
+ str q0, [dst, 16]
+ stp q0, q0, [dst, 32]
+ bic dst, dst, 63
+ sub count, dstend2, dst /* Count is now 64 too large. */
+ sub count, count, 128 /* Adjust count and bias for loop. */
+
+ .p2align 4
+8: // zva_loop
+ add dst, dst, 64
+ dc zva, dst
+ subs count, count, 64
+ b.hi 8b // zva_loop
+ stp q0, q0, [dstend2, -64]
+ stp q0, q0, [dstend2, -32]
+ ret
+
+9: // no_zva
+ sub count, dstend2, dst /* Count is 16 too large. */
+ sub dst, dst, 16 /* Dst is biased by -32. */
+ sub count, count, 64 + 16 /* Adjust count and bias for loop. */
+10: // no_zva_loop
+ stp q0, q0, [dst, 32]
+ stp q0, q0, [dst, 64]!
+ subs count, count, 64
+ b.hi 10b // no_zva_loop
+ stp q0, q0, [dstend2, -64]
+ stp q0, q0, [dstend2, -32]
+ ret
+END_COMPILERRT_FUNCTION(__arm_sc_memset)
+
+//
+// __arm_sc_memchr
+//
+
+#define srcin x0
+#define chrin w1
+#define cntin x2
+
+#define result x0
+
+#define src x3
+#define tmp x4
+#define wtmp2 w5
+#define synd x6
+#define soff x9
+#define cntrem x10
+
+#define vrepchr v0
+#define vdata1 v1
+#define vdata2 v2
+#define vhas_chr1 v3
+#define vhas_chr2 v4
+#define vrepmask v5
+#define vend v6
+
+/*
+ * Core algorithm:
+ *
+ * For each 32-byte chunk we calculate a 64-bit syndrome value, with two bits
+ * per byte. For each tuple, bit 0 is set if the relevant byte matched the
+ * requested character and bit 1 is not used (faster than using a 32bit
+ * syndrome). Since the bits in the syndrome reflect exactly the order in which
+ * things occur in the original string, counting trailing zeros allows to
+ * identify exactly which byte has matched.
+ */
+
+DEFINE_COMPILERRT_FUNCTION(__arm_sc_memchr)
+ /* Do not dereference srcin if no bytes to compare. */
+ cbz cntin, 4f
+ /*
+ * Magic constant 0x40100401 allows us to identify which lane matches
+ * the requested byte.
+ */
+ mov wtmp2, #0x0401
+ movk wtmp2, #0x4010, lsl #16
+ dup vrepchr.16b, chrin
+ /* Work with aligned 32-byte chunks */
+ bic src, srcin, #31
+ dup vrepmask.4s, wtmp2
+ ands soff, srcin, #31
+ and cntrem, cntin, #31
+ b.eq 0f
+
+ /*
+ * Input string is not 32-byte aligned. We calculate the syndrome
+ * value for the aligned 32 bytes block containing the first bytes
+ * and mask the irrelevant part.
+ */
+
+ ld1 {vdata1.16b, vdata2.16b}, [src], #32
+ sub tmp, soff, #32
+ adds cntin, cntin, tmp
+ cmeq vhas_chr1.16b, vdata1.16b, vrepchr.16b
+ cmeq vhas_chr2.16b, vdata2.16b, vrepchr.16b
+ and vhas_chr1.16b, vhas_chr1.16b, vrepmask.16b
+ and vhas_chr2.16b, vhas_chr2.16b, vrepmask.16b
+ addp vend.16b, vhas_chr1.16b, vhas_chr2.16b /* 256->128 */
+ addp vend.16b, vend.16b, vend.16b /* 128->64 */
+ mov synd, vend.d[0]
+ /* Clear the soff*2 lower bits */
+ lsl tmp, soff, #1
+ lsr synd, synd, tmp
+ lsl synd, synd, tmp
+ /* The first block can also be the last */
+ b.ls 2f
+ /* Have we found something already? */
+ cbnz synd, 3f
+
+0: // loop
+ ld1 {vdata1.16b, vdata2.16b}, [src], #32
+ subs cntin, cntin, #32
+ cmeq vhas_chr1.16b, vdata1.16b, vrepchr.16b
+ cmeq vhas_chr2.16b, vdata2.16b, vrepchr.16b
+ /* If we're out of data we finish regardless of the result */
+ b.ls 1f
+ /* Use a fast check for the termination condition */
+ orr vend.16b, vhas_chr1.16b, vhas_chr2.16b
+ addp vend.2d, vend.2d, vend.2d
+ mov synd, vend.d[0]
+ /* We're not out of data, loop if we haven't found the character */
+ cbz synd, 0b
+
+1: // end
+ /* Termination condition found, let's calculate the syndrome value */
+ and vhas_chr1.16b, vhas_chr1.16b, vrepmask.16b
+ and vhas_chr2.16b, vhas_chr2.16b, vrepmask.16b
+ addp vend.16b, vhas_chr1.16b, vhas_chr2.16b /* 256->128 */
+ addp vend.16b, vend.16b, vend.16b /* 128->64 */
+ mov synd, vend.d[0]
+ /* Only do the clear for the last possible block */
+ b.hi 3f
+
+2: // masklast
+ /* Clear the (32 - ((cntrem + soff) % 32)) * 2 upper bits */
+ add tmp, cntrem, soff
+ and tmp, tmp, #31
+ sub tmp, tmp, #32
+ neg tmp, tmp, lsl #1
+ lsl synd, synd, tmp
+ lsr synd, synd, tmp
+
+3: // tail
+ /* Count the trailing zeros using bit reversing */
+ rbit synd, synd
+ /* Compensate the last post-increment */
+ sub src, src, #32
+ /* Check that we have found a character */
+ cmp synd, #0
+ /* And count the leading zeros */
+ clz synd, synd
+ /* Compute the potential result */
+ add result, src, synd, lsr #1
+ /* Select result or NULL */
+ csel result, xzr, result, eq
+ ret
+
+4: // zero_length
+ mov result, #0
+ ret
+END_COMPILERRT_FUNCTION(__arm_sc_memchr)
+
diff --git a/compiler-rt/lib/builtins/aarch64/sme-libc-routines.c b/compiler-rt/lib/builtins/aarch64/sme-libc-routines.c
deleted file mode 100644
index 24b9b3bcb49da..0000000000000
--- a/compiler-rt/lib/builtins/aarch64/sme-libc-routines.c
+++ /dev/null
@@ -1,68 +0,0 @@
-#include <stddef.h>
-
-// The asm version uses FP registers and unaligned memory accesses. Use this on
-// targets without them.
-#if __ARM_FP == 0 || !defined(__ARM_FEATURE_UNALIGNED)
-void *__arm_sc_memset(void *dest, int c, size_t n) __arm_streaming_compatible {
- unsigned char *destp = (unsigned char *)dest;
- unsigned char c8 = (unsigned char)c;
- for (size_t i = 0; i < n; ++i)
- destp[i] = c8;
-
- return dest;
-}
-#endif
-
-const void *__arm_sc_memchr(const void *src, int c,
- size_t n) __arm_streaming_compatible {
- const unsigned char *srcp = (const unsigned char *)src;
- unsigned char c8 = (unsigned char)c;
- for (size_t i = 0; i < n; ++i)
- if (srcp[i] == c8)
- return &srcp[i];
-
- return NULL;
-}
-
-#ifndef __ARM_FEATURE_UNALIGNED
-
-static void *memcpy_fwd(void *dest, const void *src,
- size_t n) __arm_streaming_compatible {
- unsigned char *destp = (unsigned char *)dest;
- const unsigned char *srcp = (const unsigned char *)src;
-
- for (size_t i = 0; i < n; ++i)
- destp[i] = srcp[i];
- return dest;
-}
-
-static void *memcpy_rev(void *dest, const void *src,
- size_t n) __arm_streaming_compatible {
- unsigned char *destp = (unsigned char *)dest;
- const unsigned char *srcp = (const unsigned char *)src;
-
- while (n > 0) {
- --n;
- destp[n] = srcp[n];
- }
- return dest;
-}
-
-void *__arm_sc_memcpy(void *__restrict dest, const void *__restrict src,
- size_t n) __arm_streaming_compatible {
- return memcpy_fwd(dest, src, n);
-}
-
-void *__arm_sc_memmove(void *dest, const void *src,
- size_t n) __arm_streaming_compatible {
- unsigned char *destp = (unsigned char *)dest;
- const unsigned char *srcp = (const unsigned char *)src;
-
- if ((srcp > (destp + n)) || (destp > (srcp + n)))
- return __arm_sc_memcpy(dest, src, n);
- if (srcp > destp)
- return memcpy_fwd(dest, src, n);
- return memcpy_rev(dest, src, n);
-}
-
-#endif /* !defined(__ARM_FEATURE_UNALIGNED) */
More information about the llvm-commits
mailing list