[compiler-rt] [compiler-rt][asan] Add Linux HSA/ROCm GPU memory support via HSA API interceptors (PR #192240)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 26 01:51:45 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-compiler-rt-sanitizer
Author: Amit Kumar Pandey (ampandey-1995)
<details>
<summary>Changes</summary>
Add AddressSanitizer support for ROCm/HSA GPU memory on Linux by intercepting
host-side HSA APIs (init/shutdown, memory pool allocate/free, IPC attach/detach,
vmem reserve/free, memory copies, and pointer queries).
**Build / platform:**
- `SANITIZER_AMDHSA` is enabled by default on Linux (non-Android). The HSA path
is inert at runtime until `libhsa-runtime64.so` is dlopen'd; no ROCm install
is required to build compiler-rt.
- Vendored HSA headers are shared with the AMDGPU offload plugin
(`offload/plugins-nextgen/amdgpu/dynamic_hsa/`), avoiding drift and removing
the old CMake ROCm HSA/COMgr header discovery.
**Allocator architecture:**
- Keep `CombinedAllocator` host-only for all sanitizers.
- Add `DeviceCombinedAllocator` as an ASan-only wrapper that composes the host
allocator with `AmdgpuDeviceAllocatorT`. Host `malloc`/`free` still use the
inner `CombinedAllocator`; Device allocations go through a separate
`AllocateDevice()` path used by HSA interceptors.
- ROCr entry points are resolved via dlopen/dlsym on the ROCr handle after
`hsa_init`, avoiding RTLD_NEXT recursion.
- GPU-only vmem reservations (without `HSA_AMD_VMEM_ADDRESS_NO_REGISTER`) are
tracked separately via `VmemGpuReserveTracker` because the reserved VA is not
host-writable for ASan chunk metadata.
- Tests under `test/asan/TestCases/AMDGPU/` gate on Linux + ROCm runtime
detection; VMEM tests additionally require the `hsa-vmem` lit feature when
`HSA_VMEM_SUPPORTED` is set.
Co-authored-by: Brian Sumner <Brian.Sumner@<!-- -->amd.com>
Co-authored-by: Bing Ma <Bing.Ma@<!-- -->amd.com>
---
Full diff: https://github.com/llvm/llvm-project/pull/192240.diff
2 Files Affected:
- (modified) compiler-rt/CMakeLists.txt (+20)
- (modified) compiler-rt/lib/asan/asan_interceptors.cpp (+164)
``````````diff
diff --git a/compiler-rt/CMakeLists.txt b/compiler-rt/CMakeLists.txt
index 9e45a43b34287..212a5062d25ac 100644
--- a/compiler-rt/CMakeLists.txt
+++ b/compiler-rt/CMakeLists.txt
@@ -562,6 +562,26 @@ elseif(COMPILER_RT_HAS_G_FLAG)
list(APPEND SANITIZER_COMMON_CFLAGS -g)
endif()
+if(SANITIZER_AMDGPU)
+ list(APPEND SANITIZER_COMMON_CFLAGS -DSANITIZER_AMDGPU=1)
+ message(STATUS "Looking 'hsa' and 'amd_comgr' header")
+ find_path(HSA_INCLUDE NAMES hsa.h HINTS ${SANITIZER_HSA_INCLUDE_PATH} /opt/rocm/include PATH_SUFFIXES hsa)
+ if(NOT HSA_INCLUDE)
+ message(FATAL_ERROR "Required header 'hsa.h' not found in path ${HSA_INCLUDE}. Aborting SANITIZER_AMDGPU build")
+ endif()
+ message(STATUS "Found 'hsa.h' in ${HSA_INCLUDE}")
+ include_directories(${HSA_INCLUDE})
+ find_path(COMgr_INCLUDE NAMES amd_comgr.h.in HINTS ${SANITIZER_COMGR_INCLUDE_PATH} PATH_SUFFIXES amd_comgr)
+ if(NOT COMgr_INCLUDE)
+ find_path(COMgr_INCLUDE NAMES amd_comgr.h HINTS /opt/rocm/include PATH_SUFFIXES amd_comgr)
+ if(NOT COMgr_INCLUDE)
+ message(FATAL_ERROR "Required header 'amd_comgr.h/amd_comgr.h.in' not found in path ${COMgr_INCLUDE}. Aborting SANITIZER_AMDGPU build")
+ endif()
+ endif()
+ message(STATUS "Found 'amd_comgr.h.in/amd_comgr.h' in ${COMgr_INCLUDE}")
+ include_directories(${COMgr_INCLUDE})
+endif()
+
if(LLVM_ENABLE_MODULES)
# Sanitizers cannot be built with -fmodules. The interceptors intentionally
# don't include system headers, which is incompatible with modules.
diff --git a/compiler-rt/lib/asan/asan_interceptors.cpp b/compiler-rt/lib/asan/asan_interceptors.cpp
index 4e91e34f76cee..cd580c411b253 100644
--- a/compiler-rt/lib/asan/asan_interceptors.cpp
+++ b/compiler-rt/lib/asan/asan_interceptors.cpp
@@ -897,6 +897,166 @@ DEFINE_REAL(int, vfork, )
DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork, )
# endif
+# if SANITIZER_AMDGPU
+void ENSURE_HSA_INITED();
+
+INTERCEPTOR(hsa_status_t, hsa_amd_memory_pool_allocate,
+ hsa_amd_memory_pool_t memory_pool, size_t size, uint32_t flags,
+ void** ptr) {
+ AsanInitFromRtl();
+ ENSURE_HSA_INITED();
+ GET_STACK_TRACE_MALLOC;
+ return asan_hsa_amd_memory_pool_allocate(memory_pool, size, flags, ptr,
+ &stack);
+}
+
+INTERCEPTOR(hsa_status_t, hsa_amd_memory_pool_free, void* ptr) {
+ AsanInitFromRtl();
+ ENSURE_HSA_INITED();
+ GET_STACK_TRACE_FREE;
+ return asan_hsa_amd_memory_pool_free(ptr, &stack);
+}
+
+INTERCEPTOR(hsa_status_t, hsa_amd_agents_allow_access, uint32_t num_agents,
+ const hsa_agent_t* agents, const uint32_t* flags, const void* ptr) {
+ AsanInitFromRtl();
+ ENSURE_HSA_INITED();
+ GET_STACK_TRACE_FREE;
+ return asan_hsa_amd_agents_allow_access(num_agents, agents, flags, ptr,
+ &stack);
+}
+
+INTERCEPTOR(hsa_status_t, hsa_memory_copy, void* dst, const void* src,
+ size_t size) {
+ AsanInitFromRtl();
+ ENSURE_HSA_INITED();
+ if (flags()->replace_intrin) {
+ if (dst != src) {
+ CHECK_RANGES_OVERLAP("hsa_memory_copy", dst, size, src, size);
+ }
+ ASAN_READ_RANGE(nullptr, src, size);
+ ASAN_WRITE_RANGE(nullptr, dst, size);
+ }
+ return REAL(hsa_memory_copy)(dst, src, size);
+}
+
+INTERCEPTOR(hsa_status_t, hsa_amd_memory_async_copy, void* dst,
+ hsa_agent_t dst_agent, const void* src, hsa_agent_t src_agent,
+ size_t size, uint32_t num_dep_signals,
+ const hsa_signal_t* dep_signals, hsa_signal_t completion_signal) {
+ AsanInitFromRtl();
+ ENSURE_HSA_INITED();
+ if (flags()->replace_intrin) {
+ if (dst != src) {
+ CHECK_RANGES_OVERLAP("hsa_amd_memory_async_copy", dst, size, src, size);
+ }
+ ASAN_READ_RANGE(nullptr, src, size);
+ ASAN_WRITE_RANGE(nullptr, dst, size);
+ }
+ return REAL(hsa_amd_memory_async_copy)(dst, dst_agent, src, src_agent, size,
+ num_dep_signals, dep_signals,
+ completion_signal);
+}
+
+# if HSA_AMD_INTERFACE_VERSION_MINOR >= 1
+INTERCEPTOR(hsa_status_t, hsa_amd_memory_async_copy_on_engine, void* dst,
+ hsa_agent_t dst_agent, const void* src, hsa_agent_t src_agent,
+ size_t size, uint32_t num_dep_signals,
+ const hsa_signal_t* dep_signals, hsa_signal_t completion_signal,
+ hsa_amd_sdma_engine_id_t engine_id, bool force_copy_on_sdma) {
+ AsanInitFromRtl();
+ ENSURE_HSA_INITED();
+ if (flags()->replace_intrin) {
+ if (dst != src) {
+ CHECK_RANGES_OVERLAP("hsa_amd_memory_async_copy_on_engine", dst, size,
+ src, size);
+ }
+ ASAN_READ_RANGE(nullptr, src, size);
+ ASAN_WRITE_RANGE(nullptr, dst, size);
+ }
+ return REAL(hsa_amd_memory_async_copy_on_engine)(
+ dst, dst_agent, src, src_agent, size, num_dep_signals, dep_signals,
+ completion_signal, engine_id, force_copy_on_sdma);
+}
+# endif
+
+INTERCEPTOR(hsa_status_t, hsa_amd_ipc_memory_create, void* ptr, size_t len,
+ hsa_amd_ipc_memory_t* handle) {
+ AsanInitFromRtl();
+ ENSURE_HSA_INITED();
+ return asan_hsa_amd_ipc_memory_create(ptr, len, handle);
+}
+
+INTERCEPTOR(hsa_status_t, hsa_amd_ipc_memory_attach,
+ const hsa_amd_ipc_memory_t* handle, size_t len, uint32_t num_agents,
+ const hsa_agent_t* mapping_agents, void** mapped_ptr) {
+ AsanInitFromRtl();
+ ENSURE_HSA_INITED();
+ return asan_hsa_amd_ipc_memory_attach(handle, len, num_agents, mapping_agents,
+ mapped_ptr);
+}
+
+INTERCEPTOR(hsa_status_t, hsa_amd_ipc_memory_detach, void* mapped_ptr) {
+ AsanInitFromRtl();
+ ENSURE_HSA_INITED();
+ return asan_hsa_amd_ipc_memory_detach(mapped_ptr);
+}
+
+INTERCEPTOR(hsa_status_t, hsa_amd_vmem_address_reserve_align, void** ptr,
+ size_t size, uint64_t address, uint64_t alignment, uint64_t flags) {
+ AsanInitFromRtl();
+ ENSURE_HSA_INITED();
+ GET_STACK_TRACE_MALLOC;
+ return asan_hsa_amd_vmem_address_reserve_align(ptr, size, address, alignment,
+ flags, &stack);
+}
+
+INTERCEPTOR(hsa_status_t, hsa_amd_vmem_address_free, void* ptr, size_t size) {
+ AsanInitFromRtl();
+ ENSURE_HSA_INITED();
+ GET_STACK_TRACE_FREE;
+ return asan_hsa_amd_vmem_address_free(ptr, size, &stack);
+}
+
+INTERCEPTOR(hsa_status_t, hsa_amd_pointer_info, const void* ptr,
+ hsa_amd_pointer_info_t* info, void* (*alloc)(size_t),
+ uint32_t* num_agents_accessible, hsa_agent_t** accessible) {
+ AsanInitFromRtl();
+ ENSURE_HSA_INITED();
+ return asan_hsa_amd_pointer_info(ptr, info, alloc, num_agents_accessible,
+ accessible);
+}
+
+INTERCEPTOR(hsa_status_t, hsa_init) {
+ AsanInitFromRtl();
+ ENSURE_HSA_INITED();
+ return asan_hsa_init();
+}
+
+void InitializeAmdgpuInterceptors() {
+ ASAN_INTERCEPT_FUNC(hsa_init);
+ ASAN_INTERCEPT_FUNC(hsa_memory_copy);
+ ASAN_INTERCEPT_FUNC(hsa_amd_memory_pool_allocate);
+ ASAN_INTERCEPT_FUNC(hsa_amd_memory_pool_free);
+ ASAN_INTERCEPT_FUNC(hsa_amd_agents_allow_access);
+ ASAN_INTERCEPT_FUNC(hsa_amd_memory_async_copy);
+# if HSA_AMD_INTERFACE_VERSION_MINOR >= 1
+ ASAN_INTERCEPT_FUNC(hsa_amd_memory_async_copy_on_engine);
+# endif
+ ASAN_INTERCEPT_FUNC(hsa_amd_ipc_memory_create);
+ ASAN_INTERCEPT_FUNC(hsa_amd_ipc_memory_attach);
+ ASAN_INTERCEPT_FUNC(hsa_amd_ipc_memory_detach);
+ ASAN_INTERCEPT_FUNC(hsa_amd_vmem_address_reserve_align);
+ ASAN_INTERCEPT_FUNC(hsa_amd_vmem_address_free);
+ ASAN_INTERCEPT_FUNC(hsa_amd_pointer_info);
+}
+
+void ENSURE_HSA_INITED() {
+ if (!REAL(hsa_init))
+ InitializeAmdgpuInterceptors();
+}
+# endif
+
// ---------------------- InitializeAsanInterceptors ---------------- {{{1
namespace __asan {
void InitializeAsanInterceptors() {
@@ -1013,6 +1173,10 @@ void InitializeAsanInterceptors() {
ASAN_INTERCEPT_FUNC(vfork);
# endif
+# if SANITIZER_AMDGPU
+ InitializeAmdgpuInterceptors();
+# endif
+
VReport(1, "AddressSanitizer: libc interceptors initialized\n");
}
``````````
</details>
https://github.com/llvm/llvm-project/pull/192240
More information about the llvm-commits
mailing list