[libc-commits] [libc] [libc][MVE] implement MVE accelerated memcmp (PR #224857)
Schrodinger ZHU Yifan via libc-commits
libc-commits at lists.llvm.org
Thu Oct 1 06:30:59 PDT 2026
https://github.com/SchrodingerZhu updated https://github.com/llvm/llvm-project/pull/224857
>From 476c2003c521d981c7e4c28cb0c05059d7ea4841 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <i at zhuyi.fan>
Date: Sat, 19 Sep 2026 16:23:34 -0400
Subject: [PATCH 1/4] [libc][MVE] implement MVE accelerated memcmp
---
libc/src/string/memory_utils/CMakeLists.txt | 2 +
.../string/memory_utils/arm/inline_memcmp.h | 64 +++++++++++++++++++
libc/src/string/memory_utils/inline_memcmp.h | 3 +
3 files changed, 69 insertions(+)
create mode 100644 libc/src/string/memory_utils/arm/inline_memcmp.h
diff --git a/libc/src/string/memory_utils/CMakeLists.txt b/libc/src/string/memory_utils/CMakeLists.txt
index 510546d79988aed..17da91e15a686d8 100644
--- a/libc/src/string/memory_utils/CMakeLists.txt
+++ b/libc/src/string/memory_utils/CMakeLists.txt
@@ -10,6 +10,7 @@ add_header_library(
arm/common.h
arm/inline_memcpy.h
arm/inline_memset.h
+ arm/inline_memcmp.h
generic/aligned_access.h
generic/byte_per_byte.h
inline_bcmp.h
@@ -44,6 +45,7 @@ add_header_library(
libc.src.__support.macros.optimization
libc.src.__support.macros.properties.architectures
libc.src.__support.macros.properties.compiler
+ libc.src.__support.macros.properties.cpu_features
)
add_header_library(
diff --git a/libc/src/string/memory_utils/arm/inline_memcmp.h b/libc/src/string/memory_utils/arm/inline_memcmp.h
new file mode 100644
index 000000000000000..96f2b906f2bb426
--- /dev/null
+++ b/libc/src/string/memory_utils/arm/inline_memcmp.h
@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the memcmp implementations for arm.
+///
+///===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ARM_INLINE_MEMCMP_H
+#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ARM_INLINE_MEMCMP_H
+
+#include "src/__support/CPP/bit.h"
+#include "src/__support/macros/attributes.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/cpu_features.h"
+#include "src/__support/math_extras.h"
+#include "src/string/memory_utils/generic/byte_per_byte.h"
+#include "src/string/memory_utils/utils.h"
+
+#if defined(LIBC_TARGET_CPU_HAS_MVE)
+#include <arm_mve.h>
+#endif
+
+namespace LIBC_NAMESPACE_DECL {
+// We limit code size and don't aggressively expand the loop to multiway
+// parallel comparison.
+[[maybe_unused]] LIBC_INLINE MemcmpReturnType inline_memcmp_arm(CPtr p1,
+ CPtr p2,
+ size_t count) {
+#if defined(LIBC_TARGET_CPU_HAS_MVE)
+ // Cast to raw address to avoid ub:expr.add.out.of.bounds
+ uintptr_t p1_addr = cpp::bit_cast<uintptr_t>(p1);
+ uintptr_t p2_addr = cpp::bit_cast<uintptr_t>(p2);
+ while (count != 0) {
+ // Predication handles the final partial vector without reading past count.
+ mve_pred16_t active = vctp8q(count);
+ uint8x16_t a = vldrbq_z_u8(cpp::bit_cast<const uint8_t *>(p1_addr), active);
+ uint8x16_t b = vldrbq_z_u8(cpp::bit_cast<const uint8_t *>(p2_addr), active);
+ unsigned mismatches = vcmpneq_m_u8(a, b, active);
+ if (mismatches != 0) {
+ const size_t offset = cpp::countr_zero(mismatches);
+ return static_cast<int32_t>(cpp::bit_cast<CPtr>(p1_addr)[offset]) -
+ static_cast<int32_t>(cpp::bit_cast<CPtr>(p2_addr)[offset]);
+ }
+ // optimistically increase the address
+ p1_addr += 16;
+ p2_addr += 16;
+ if (sub_overflow(count, size_t{16}, count))
+ break;
+ }
+ return MemcmpReturnType::zero();
+#else
+ return inline_memcmp_byte_per_byte(p1, p2, count);
+#endif
+}
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_ARM_INLINE_MEMCMP_H
diff --git a/libc/src/string/memory_utils/inline_memcmp.h b/libc/src/string/memory_utils/inline_memcmp.h
index a98f7ac51648e8e..ac071f2ac230995 100644
--- a/libc/src/string/memory_utils/inline_memcmp.h
+++ b/libc/src/string/memory_utils/inline_memcmp.h
@@ -21,6 +21,9 @@
#elif defined(LIBC_TARGET_ARCH_IS_X86)
#include "src/string/memory_utils/x86_64/inline_memcmp.h"
#define LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP inline_memcmp_x86
+#elif defined(LIBC_TARGET_ARCH_IS_ARM)
+#include "src/string/memory_utils/arm/inline_memcmp.h"
+#define LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP inline_memcmp_arm
#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
#include "src/string/memory_utils/aarch64/inline_memcmp.h"
#define LIBC_SRC_STRING_MEMORY_UTILS_MEMCMP inline_memcmp_aarch64_dispatch
>From e2e4da15635ac15e69c7ce952cd1e3313b452aca Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <i at zhuyi.fan>
Date: Tue, 22 Sep 2026 19:17:48 -0400
Subject: [PATCH 2/4] migrate to vector instrinsics
---
libc/src/string/memory_utils/CMakeLists.txt | 1 +
.../string/memory_utils/arm/inline_memcmp.h | 19 ++++++++++++-------
2 files changed, 13 insertions(+), 7 deletions(-)
diff --git a/libc/src/string/memory_utils/CMakeLists.txt b/libc/src/string/memory_utils/CMakeLists.txt
index 17da91e15a686d8..2446cef3df6a758 100644
--- a/libc/src/string/memory_utils/CMakeLists.txt
+++ b/libc/src/string/memory_utils/CMakeLists.txt
@@ -39,6 +39,7 @@ add_header_library(
libc.src.__support.common
libc.src.__support.CPP.bit
libc.src.__support.CPP.cstddef
+ libc.src.__support.CPP.simd
libc.src.__support.CPP.type_traits
libc.src.__support.macros.config
libc.src.__support.macros.is_defined
diff --git a/libc/src/string/memory_utils/arm/inline_memcmp.h b/libc/src/string/memory_utils/arm/inline_memcmp.h
index 96f2b906f2bb426..a511368a4d3e450 100644
--- a/libc/src/string/memory_utils/arm/inline_memcmp.h
+++ b/libc/src/string/memory_utils/arm/inline_memcmp.h
@@ -23,7 +23,7 @@
#include "src/string/memory_utils/utils.h"
#if defined(LIBC_TARGET_CPU_HAS_MVE)
-#include <arm_mve.h>
+#include "src/__support/CPP/simd.h"
#endif
namespace LIBC_NAMESPACE_DECL {
@@ -33,17 +33,22 @@ namespace LIBC_NAMESPACE_DECL {
CPtr p2,
size_t count) {
#if defined(LIBC_TARGET_CPU_HAS_MVE)
+ using Bytes = cpp::simd<uint8_t, 16>;
+ using Mask = cpp::simd<bool, 16>;
+ cpp::simd<uint32_t, 16> lanes = cpp::iota<uint32_t, 16>();
// Cast to raw address to avoid ub:expr.add.out.of.bounds
uintptr_t p1_addr = cpp::bit_cast<uintptr_t>(p1);
uintptr_t p2_addr = cpp::bit_cast<uintptr_t>(p2);
while (count != 0) {
// Predication handles the final partial vector without reading past count.
- mve_pred16_t active = vctp8q(count);
- uint8x16_t a = vldrbq_z_u8(cpp::bit_cast<const uint8_t *>(p1_addr), active);
- uint8x16_t b = vldrbq_z_u8(cpp::bit_cast<const uint8_t *>(p2_addr), active);
- unsigned mismatches = vcmpneq_m_u8(a, b, active);
- if (mismatches != 0) {
- const size_t offset = cpp::countr_zero(mismatches);
+ Mask active = cpp::simd_cast<bool>(lanes < count);
+ Bytes a =
+ cpp::load_masked<Bytes>(active, cpp::bit_cast<CPtr>(p1_addr), Bytes{});
+ Bytes b =
+ cpp::load_masked<Bytes>(active, cpp::bit_cast<CPtr>(p2_addr), Bytes{});
+ Mask mismatches = cpp::simd_cast<bool>(a != b) & active;
+ if (cpp::any_of(mismatches)) {
+ size_t offset = cpp::find_first_set(mismatches);
return static_cast<int32_t>(cpp::bit_cast<CPtr>(p1_addr)[offset]) -
static_cast<int32_t>(cpp::bit_cast<CPtr>(p2_addr)[offset]);
}
>From 6fb4b866f1bde800e50ea00bc90368395e6d0181 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <i at zhuyi.fan>
Date: Tue, 22 Sep 2026 19:37:39 -0400
Subject: [PATCH 3/4] fix macro guards
---
libc/src/string/memory_utils/arm/inline_memcmp.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libc/src/string/memory_utils/arm/inline_memcmp.h b/libc/src/string/memory_utils/arm/inline_memcmp.h
index a511368a4d3e450..43ee8b79d27c7dd 100644
--- a/libc/src/string/memory_utils/arm/inline_memcmp.h
+++ b/libc/src/string/memory_utils/arm/inline_memcmp.h
@@ -32,7 +32,7 @@ namespace LIBC_NAMESPACE_DECL {
[[maybe_unused]] LIBC_INLINE MemcmpReturnType inline_memcmp_arm(CPtr p1,
CPtr p2,
size_t count) {
-#if defined(LIBC_TARGET_CPU_HAS_MVE)
+#if defined(LIBC_TARGET_CPU_HAS_MVE) && LIBC_HAS_VECTOR_TYPE
using Bytes = cpp::simd<uint8_t, 16>;
using Mask = cpp::simd<bool, 16>;
cpp::simd<uint32_t, 16> lanes = cpp::iota<uint32_t, 16>();
>From c9a91ca76b6d3ba291e149f8757dcd055b1902a2 Mon Sep 17 00:00:00 2001
From: Schrodinger ZHU Yifan <i at zhuyi.fan>
Date: Thu, 1 Oct 2026 09:30:28 -0400
Subject: [PATCH 4/4] align inclusion guard
---
libc/src/string/memory_utils/arm/inline_memcmp.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libc/src/string/memory_utils/arm/inline_memcmp.h b/libc/src/string/memory_utils/arm/inline_memcmp.h
index 43ee8b79d27c7dd..853616eac209b72 100644
--- a/libc/src/string/memory_utils/arm/inline_memcmp.h
+++ b/libc/src/string/memory_utils/arm/inline_memcmp.h
@@ -22,7 +22,7 @@
#include "src/string/memory_utils/generic/byte_per_byte.h"
#include "src/string/memory_utils/utils.h"
-#if defined(LIBC_TARGET_CPU_HAS_MVE)
+#if defined(LIBC_TARGET_CPU_HAS_MVE) && LIBC_HAS_VECTOR_TYPE
#include "src/__support/CPP/simd.h"
#endif
More information about the libc-commits
mailing list