[compiler-rt] [ASan][AIX] Intercept __linux_vec_malloc/__linux_vec_calloc/__linux_realloc (PR #209359)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 20:58:50 PDT 2026
https://github.com/midhuncodes7 updated https://github.com/llvm/llvm-project/pull/209359
>From c30edde6e3f3f1241a9c88f0a9b9aaeca549c6b8 Mon Sep 17 00:00:00 2001
From: Midhunesh <midhunesh.p at ibm.com>
Date: Mon, 13 Jul 2026 18:43:42 +0530
Subject: [PATCH 1/3] alignment preserve linux vec
---
.../lib/asan/AIX/asan.link_with_main_exec.txt | 6 +++
compiler-rt/lib/asan/asan_malloc_linux.cpp | 25 ++++++++++
.../asan/TestCases/AIX/linux_vec_alloc.cpp | 46 +++++++++++++++++++
3 files changed, 77 insertions(+)
create mode 100644 compiler-rt/test/asan/TestCases/AIX/linux_vec_alloc.cpp
diff --git a/compiler-rt/lib/asan/AIX/asan.link_with_main_exec.txt b/compiler-rt/lib/asan/AIX/asan.link_with_main_exec.txt
index 5efc48c262369..0e02267803fbd 100644
--- a/compiler-rt/lib/asan/AIX/asan.link_with_main_exec.txt
+++ b/compiler-rt/lib/asan/AIX/asan.link_with_main_exec.txt
@@ -113,3 +113,9 @@ __asan_poison_stack_memory
__asan_unpoison_stack_memory
__asan_option_detect_stack_use_after_return
__asan_shadow_memory_dynamic_address
+__interceptor___linux_vec_malloc
+__linux_vec_malloc
+__interceptor___linux_vec_calloc
+__linux_vec_calloc
+__interceptor___linux_realloc
+__linux_realloc
diff --git a/compiler-rt/lib/asan/asan_malloc_linux.cpp b/compiler-rt/lib/asan/asan_malloc_linux.cpp
index 35bf55da1c66b..9ac80a0b28831 100644
--- a/compiler-rt/lib/asan/asan_malloc_linux.cpp
+++ b/compiler-rt/lib/asan/asan_malloc_linux.cpp
@@ -84,6 +84,31 @@ INTERCEPTOR(void, free_aligned_sized, void* ptr, uptr alignment, uptr size) {
# endif // SANITIZER_INTERCEPT_FREE_ALIGNED_SIZED
# if SANITIZER_AIX
+// __linux_vec_malloc, __linux_vec_calloc, and __linux_realloc are XL compiler
+// internal symbols emitted instead of the standard vec_malloc/vec_calloc/realloc
+// names. Intercept them so that ASan tracks allocations made through the XL
+// runtime.
+INTERCEPTOR(void*, __linux_vec_malloc, uptr size) {
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Allocate(size, 16);
+ GET_STACK_TRACE_MALLOC;
+ return asan_vec_malloc(size, &stack);
+}
+
+INTERCEPTOR(void*, __linux_vec_calloc, uptr nmemb, uptr size) {
+ if (DlsymAlloc::Use())
+ return DlsymAlloc::Callocate(nmemb, size, 16);
+ GET_STACK_TRACE_MALLOC;
+ return asan_vec_calloc(nmemb, size, &stack);
+}
+
+INTERCEPTOR(void*, __linux_realloc, void* ptr, uptr size) {
+ if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
+ return DlsymAlloc::Realloc(ptr, size, kWordSize);
+ GET_STACK_TRACE_MALLOC;
+ return asan_vec_realloc(ptr, size, &stack);
+}
+
// Unlike malloc, vec_malloc must return memory aligned to 16 bytes.
INTERCEPTOR(void*, vec_malloc, uptr size) {
if (DlsymAlloc::Use())
diff --git a/compiler-rt/test/asan/TestCases/AIX/linux_vec_alloc.cpp b/compiler-rt/test/asan/TestCases/AIX/linux_vec_alloc.cpp
new file mode 100644
index 0000000000000..8b694a23cb839
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/AIX/linux_vec_alloc.cpp
@@ -0,0 +1,46 @@
+// Verify __linux_vec_malloc, __linux_vec_calloc, and __linux_realloc
+// interceptors detect out-of-bounds accesses.
+
+// RUN: %clangxx_asan -O0 %s -o %t
+// RUN: not %run %t linux_vec_malloc 2>&1 | FileCheck %s --check-prefix=CHECK-MALLOC
+// RUN: not %run %t linux_vec_calloc 2>&1 | FileCheck %s --check-prefix=CHECK-CALLOC
+// RUN: not %run %t linux_realloc 2>&1 | FileCheck %s --check-prefix=CHECK-REALLOC
+
+#include <stdlib.h>
+#include <string.h>
+
+extern "C" {
+void *__linux_vec_malloc(unsigned long size);
+void *__linux_vec_calloc(unsigned long nmemb, unsigned long size);
+void *__linux_realloc(void *ptr, unsigned long size);
+}
+
+int main(int argc, char **argv) {
+ if (argc != 2)
+ return 1;
+
+ char *p;
+ if (strcmp(argv[1], "linux_vec_malloc") == 0)
+ p = (char *)__linux_vec_malloc(10);
+ // CHECK-MALLOC: {{READ of size 1 at 0x.* thread T0}}
+ // CHECK-MALLOC: {{0x.* is located 0 bytes after 10-byte region}}
+ // CHECK-MALLOC: {{0x.* in .*__linux_vec_malloc}}
+ else if (strcmp(argv[1], "linux_vec_calloc") == 0)
+ p = (char *)__linux_vec_calloc(10, 1);
+ // CHECK-CALLOC: {{READ of size 1 at 0x.* thread T0}}
+ // CHECK-CALLOC: {{0x.* is located 0 bytes after 10-byte region}}
+ // CHECK-CALLOC: {{0x.* in .*__linux_vec_calloc}}
+ else if (strcmp(argv[1], "linux_realloc") == 0) {
+ char *orig = (char *)__linux_vec_malloc(5);
+ p = (char *)__linux_realloc(orig, 10);
+ }
+ // CHECK-REALLOC: {{READ of size 1 at 0x.* thread T0}}
+ // CHECK-REALLOC: {{0x.* is located 0 bytes after 10-byte region}}
+ // CHECK-REALLOC: {{0x.* in .*__linux_realloc}}
+ else
+ return 1;
+
+ char x = p[10];
+ free(p);
+ return x;
+}
>From 158865c166937c0e42e39e57ce249fd8387807b0 Mon Sep 17 00:00:00 2001
From: Midhunesh <midhunesh.p at ibm.com>
Date: Mon, 13 Jul 2026 21:15:24 +0530
Subject: [PATCH 2/3] Intercept
__linux_vec_malloc/__linux_vec_calloc/__linux_realloc
---
compiler-rt/lib/asan/asan_malloc_linux.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/compiler-rt/lib/asan/asan_malloc_linux.cpp b/compiler-rt/lib/asan/asan_malloc_linux.cpp
index 9ac80a0b28831..7ff10ab708a0e 100644
--- a/compiler-rt/lib/asan/asan_malloc_linux.cpp
+++ b/compiler-rt/lib/asan/asan_malloc_linux.cpp
@@ -106,7 +106,7 @@ INTERCEPTOR(void*, __linux_realloc, void* ptr, uptr size) {
if (DlsymAlloc::Use() || DlsymAlloc::PointerIsMine(ptr))
return DlsymAlloc::Realloc(ptr, size, kWordSize);
GET_STACK_TRACE_MALLOC;
- return asan_vec_realloc(ptr, size, &stack);
+ return asan_realloc(ptr, size, &stack);
}
// Unlike malloc, vec_malloc must return memory aligned to 16 bytes.
>From d512a334d0fb938f24449c4b080b7c7b99ba6225 Mon Sep 17 00:00:00 2001
From: Midhunesh <midhuensh.p at ibm.com>
Date: Mon, 13 Jul 2026 23:58:34 -0400
Subject: [PATCH 3/3] Fix formatting
---
compiler-rt/lib/asan/asan_malloc_linux.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/compiler-rt/lib/asan/asan_malloc_linux.cpp b/compiler-rt/lib/asan/asan_malloc_linux.cpp
index 6f87dbb1eb798..43cab92ee46d8 100644
--- a/compiler-rt/lib/asan/asan_malloc_linux.cpp
+++ b/compiler-rt/lib/asan/asan_malloc_linux.cpp
@@ -85,9 +85,9 @@ INTERCEPTOR(void, free_aligned_sized, void* ptr, uptr alignment, uptr size) {
# if SANITIZER_AIX
// __linux_vec_malloc, __linux_vec_calloc, and __linux_realloc are XL compiler
-// internal symbols emitted instead of the standard vec_malloc/vec_calloc/realloc
-// names. Intercept them so that ASan tracks allocations made through the XL
-// runtime.
+// internal symbols emitted instead of the standard
+// vec_malloc/vec_calloc/realloc names. Intercept them so that ASan tracks
+// allocations made through the XL runtime.
INTERCEPTOR(void*, __linux_vec_malloc, uptr size) {
if (DlsymAlloc::Use())
return DlsymAlloc::Allocate(size, 16);
More information about the llvm-commits
mailing list