[libc-commits] [libc] [libc] Annotate RPC members as global pointers (PR #215412)
Joseph Huber via libc-commits
libc-commits at lists.llvm.org
Mon Aug 10 14:55:43 PDT 2026
https://github.com/jhuber6 created https://github.com/llvm/llvm-project/pull/215412
Summary:
These are currently all routed through the generic interface. Doing this
should safe a handful of instructions and make the intent clearer.
Basically, makes it nicer on the optimizer and reduces the number of
wait counts while effectively being NFC because this address space is
where these always lived..
>From 4869ea6924c32c5b30558a1363e16cc9cfa4dee3 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn at outlook.com>
Date: Mon, 10 Aug 2026 16:50:00 -0500
Subject: [PATCH] [libc] Annotate RPC members as global pointers
Summary:
These are currently all routed through the generic interface. Doing this
should safe a handful of instructions and make the intent clearer.
Basically, makes it nicer on the optimizer and reduces the number of
wait counts while effectively being NFC because this address space is
where these always lived..
---
libc/shared/rpc.h | 52 +++++++++++++++++++++---------------------
libc/shared/rpc_util.h | 18 +++++++++++----
2 files changed, 40 insertions(+), 30 deletions(-)
diff --git a/libc/shared/rpc.h b/libc/shared/rpc.h
index 2e899354fe592..50ff9439778d7 100644
--- a/libc/shared/rpc.h
+++ b/libc/shared/rpc.h
@@ -74,8 +74,8 @@ static_assert(sizeof(Buffer) == 64, "Buffer size mismatch");
/// A target specific struct containing a doorbell to wake the server-side
/// thread.
struct alignas(64) Doorbell {
- uint64_t *value;
- uint64_t *mailbox;
+ RPC_GLOBAL uint64_t *value;
+ RPC_GLOBAL uint64_t *mailbox;
uint32_t event_id;
};
@@ -112,26 +112,26 @@ template <bool Invert> struct Process {
RPC_ATTRS ~Process() = default;
const uint32_t port_count = 0;
- Doorbell *const doorbell = nullptr;
- const uint32_t *const inbox = nullptr;
- uint32_t *const outbox = nullptr;
- Header *const header = nullptr;
- Buffer *const packet = nullptr;
+ RPC_GLOBAL Doorbell *const doorbell = nullptr;
+ RPC_GLOBAL const uint32_t *const inbox = nullptr;
+ RPC_GLOBAL uint32_t *const outbox = nullptr;
+ RPC_GLOBAL Header *const header = nullptr;
+ RPC_GLOBAL Buffer *const packet = nullptr;
static constexpr uint64_t NUM_BITS_IN_WORD = sizeof(uint32_t) * 8;
uint32_t lock[MAX_PORT_COUNT / NUM_BITS_IN_WORD] = {0};
+ // The buffer is supplied by the host unqualified, so the casts below are
+ // C-style as they need to change the address space.
RPC_ATTRS Process(uint32_t port_count, void *buffer)
- : port_count(port_count), doorbell(reinterpret_cast<Doorbell *>(
- advance(buffer, doorbell_offset()))),
- inbox(reinterpret_cast<uint32_t *>(
- advance(buffer, inbox_offset(port_count)))),
- outbox(reinterpret_cast<uint32_t *>(
- advance(buffer, outbox_offset(port_count)))),
- header(reinterpret_cast<Header *>(
- advance(buffer, header_offset(port_count)))),
- packet(reinterpret_cast<Buffer *>(
- advance(buffer, buffer_offset(port_count)))) {}
+ : port_count(port_count),
+ doorbell((RPC_GLOBAL Doorbell *)advance(buffer, doorbell_offset())),
+ inbox((RPC_GLOBAL uint32_t *)advance(buffer, inbox_offset(port_count))),
+ outbox(
+ (RPC_GLOBAL uint32_t *)advance(buffer, outbox_offset(port_count))),
+ header((RPC_GLOBAL Header *)advance(buffer, header_offset(port_count))),
+ packet(
+ (RPC_GLOBAL Buffer *)advance(buffer, buffer_offset(port_count))) {}
/// Allocate a memory buffer sufficient to store the following equivalent
/// representation in memory.
@@ -225,7 +225,7 @@ template <bool Invert> struct Process {
/// The packet is a linearly allocated array of buffers used to communicate
/// with the other process. This function returns the appropriate slot in this
/// array such that the process can operate on an entire warp or wavefront.
- RPC_ATTRS Buffer *get_packet(uint32_t index, uint32_t lane_size) {
+ RPC_ATTRS RPC_GLOBAL Buffer *get_packet(uint32_t index, uint32_t lane_size) {
return &packet[index * lane_size];
}
@@ -356,7 +356,7 @@ template <bool Invert> struct Process {
/// Invokes a function across every active buffer across the total lane size.
template <typename F>
RPC_ATTRS static void invoke_rpc(F &&fn, uint32_t lane_size, uint64_t lane_mask,
- Buffer *slot) {
+ RPC_GLOBAL Buffer *slot) {
if constexpr (is_process_gpu()) {
fn(&slot[rpc::get_lane_id()], rpc::get_lane_id());
} else {
@@ -527,7 +527,7 @@ template <bool T>
template <typename W>
RPC_ATTRS void Port<T>::recv_and_send(W work) {
recv(work);
- send([](Buffer *, uint32_t) { /* no-op */ });
+ send([](RPC_GLOBAL Buffer *, uint32_t) { /* no-op */ });
}
/// Helper routine to simplify the interface when sending from the GPU using
@@ -544,7 +544,7 @@ RPC_ATTRS void Port<T>::send_n(const void *src, uint64_t size) {
template <bool T>
RPC_ATTRS void Port<T>::send_n(const void *const *src, uint64_t *size) {
uint64_t num_sends = 0;
- send([&](Buffer *buffer, uint32_t id) {
+ send([&](RPC_GLOBAL Buffer *buffer, uint32_t id) {
reinterpret_cast<uint64_t *>(buffer->data)[0] = lane_value(size, id);
num_sends = is_process_gpu() ? lane_value(size, id)
: rpc::max(lane_value(size, id), num_sends);
@@ -557,7 +557,7 @@ RPC_ATTRS void Port<T>::send_n(const void *const *src, uint64_t *size) {
uint64_t idx = sizeof(Buffer::data) - sizeof(uint64_t);
uint64_t mask = process.header[index].mask;
while (rpc::ballot(mask, idx < num_sends)) {
- send([=](Buffer *buffer, uint32_t id) {
+ send([=](RPC_GLOBAL Buffer *buffer, uint32_t id) {
uint64_t len = lane_value(size, id) - idx > sizeof(Buffer::data)
? sizeof(Buffer::data)
: lane_value(size, id) - idx;
@@ -575,7 +575,7 @@ template <bool T>
template <typename A>
RPC_ATTRS void Port<T>::recv_n(void **dst, uint64_t *size, A &&alloc) {
uint64_t num_recvs = 0;
- recv([&](Buffer *buffer, uint32_t id) {
+ recv([&](RPC_GLOBAL Buffer *buffer, uint32_t id) {
lane_value(size, id) = reinterpret_cast<uint64_t *>(buffer->data)[0];
lane_value(dst, id) =
reinterpret_cast<uint8_t *>(alloc(lane_value(size, id)));
@@ -590,7 +590,7 @@ RPC_ATTRS void Port<T>::recv_n(void **dst, uint64_t *size, A &&alloc) {
uint64_t idx = sizeof(Buffer::data) - sizeof(uint64_t);
uint64_t mask = process.header[index].mask;
while (rpc::ballot(mask, idx < num_recvs)) {
- recv([=](Buffer *buffer, uint32_t id) {
+ recv([=](RPC_GLOBAL Buffer *buffer, uint32_t id) {
uint64_t len = lane_value(size, id) - idx > sizeof(Buffer::data)
? sizeof(Buffer::data)
: lane_value(size, id) - idx;
@@ -607,7 +607,7 @@ template <typename Ty>
RPC_ATTRS void Port<T>::send_n(const Ty *src) {
for (uint64_t idx = 0; idx < sizeof(Ty); idx += sizeof(Buffer::data)) {
const uint64_t bytes = rpc::min(sizeof(Ty) - idx, sizeof(Buffer::data));
- send([&](Buffer *buffer, uint32_t id) {
+ send([&](RPC_GLOBAL Buffer *buffer, uint32_t id) {
rpc_memcpy(buffer->data, advance(&lane_value(src, id), idx), bytes);
});
}
@@ -619,7 +619,7 @@ template <typename Ty>
RPC_ATTRS void Port<T>::recv_n(Ty *dst) {
for (uint64_t idx = 0; idx < sizeof(Ty); idx += sizeof(Buffer::data)) {
const uint64_t bytes = rpc::min(sizeof(Ty) - idx, sizeof(Buffer::data));
- recv([&](Buffer *buffer, uint32_t id) {
+ recv([&](RPC_GLOBAL Buffer *buffer, uint32_t id) {
rpc_memcpy(advance(&lane_value(dst, id), idx), buffer->data, bytes);
});
}
diff --git a/libc/shared/rpc_util.h b/libc/shared/rpc_util.h
index 09a97c63b562f..5fabeb8069f5b 100644
--- a/libc/shared/rpc_util.h
+++ b/libc/shared/rpc_util.h
@@ -32,6 +32,14 @@
#endif
#endif
+#ifndef RPC_GLOBAL
+#ifdef RPC_TARGET_IS_GPU
+#define RPC_GLOBAL __gpu_global
+#else
+#define RPC_GLOBAL
+#endif
+#endif
+
namespace rpc {
template <typename T> struct type_identity {
@@ -452,19 +460,21 @@ template <typename T, typename U> RPC_ATTRS T *advance(T *ptr, U bytes) {
}
/// Wrapper around the optimal memory copy implementation for the target.
-RPC_ATTRS void rpc_memcpy(void *dst, const void *src, uint64_t count) {
+template <typename D, typename S>
+RPC_ATTRS void rpc_memcpy(D *dst, S *src, uint64_t count) {
#if __has_builtin(__builtin_memcpy)
if (count)
__builtin_memcpy(dst, src, count);
#else
+ // The casts are C-style as they may need to change the address space.
for (uint64_t i = 0; i < count; ++i)
- static_cast<uint8_t *>(dst)[i] = static_cast<const uint8_t *>(src)[i];
+ ((uint8_t *)dst)[i] = ((const uint8_t *)src)[i];
#endif
}
/// Minimal string length function.
-RPC_ATTRS constexpr uint64_t string_length(const char *s) {
- const char *end = s;
+template <typename T> RPC_ATTRS constexpr uint64_t string_length(const T *s) {
+ const T *end = s;
for (; *end != '\0'; ++end)
;
return static_cast<uint64_t>(end - s + 1);
More information about the libc-commits
mailing list