[compiler-rt] [sanitizers][windows] Add report on intercept fail to more Rtl* functions. (PR #163818)

[via llvm-commits]

Bernhard =?utf-8?q?Übelacker?= <bernhardu at mailbox.org>
Message-ID:
In-Reply-To: <llvm.org/llvm/llvm-project/pull/163818 at github.com>


https://github.com/bernhardu updated https://github.com/llvm/llvm-project/pull/163818

>From 6cd25d9149c1e014c53ef7f823d7863b9ef3f046 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bernhard=20=C3=9Cbelacker?= <bernhardu at mailbox.org>
Date: Thu, 16 Oct 2025 18:04:56 +0200
Subject: [PATCH 1/2] [sanitizers][windows] Add report on intercept fail to
 more Rtl* functions.

That way failed interceptions of these Rtl* functions
will report a message with verbosity >= 1.
---
 compiler-rt/lib/asan/asan_malloc_win.cpp | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/compiler-rt/lib/asan/asan_malloc_win.cpp b/compiler-rt/lib/asan/asan_malloc_win.cpp
index ea6f7dfaa08cf..a122834007eab 100644
--- a/compiler-rt/lib/asan/asan_malloc_win.cpp
+++ b/compiler-rt/lib/asan/asan_malloc_win.cpp
@@ -485,10 +485,11 @@ INTERCEPTOR_WINAPI(void*, RtlReAllocateHeap, HANDLE HeapHandle, DWORD Flags,
 
 namespace __asan {
 
-static void TryToOverrideFunction(const char *fname, uptr new_func) {
+static void TryToOverrideFunction(const char *fname, uptr new_func,
+                                  uptr *orig_old_func = 0) {
   // Failure here is not fatal. The CRT may not be present, and different CRT
   // versions use different symbols.
-  if (!__interception::OverrideFunction(fname, new_func))
+  if (!__interception::OverrideFunction(fname, new_func, orig_old_func))
     VPrintf(2, "Failed to override function %s\n", fname);
 }
 
@@ -519,16 +520,14 @@ void ReplaceSystemMalloc() {
     ASAN_INTERCEPT_FUNC(HeapAlloc);
 
     // Undocumented functions must be intercepted by name, not by symbol.
-    __interception::OverrideFunction("RtlSizeHeap", (uptr)WRAP(RtlSizeHeap),
-                                     (uptr *)&REAL(RtlSizeHeap));
-    __interception::OverrideFunction("RtlFreeHeap", (uptr)WRAP(RtlFreeHeap),
-                                     (uptr *)&REAL(RtlFreeHeap));
-    __interception::OverrideFunction("RtlReAllocateHeap",
-                                     (uptr)WRAP(RtlReAllocateHeap),
-                                     (uptr *)&REAL(RtlReAllocateHeap));
-    __interception::OverrideFunction("RtlAllocateHeap",
-                                     (uptr)WRAP(RtlAllocateHeap),
-                                     (uptr *)&REAL(RtlAllocateHeap));
+    TryToOverrideFunction("RtlSizeHeap", (uptr)WRAP(RtlSizeHeap),
+                          (uptr *)&REAL(RtlSizeHeap));
+    TryToOverrideFunction("RtlFreeHeap", (uptr)WRAP(RtlFreeHeap),
+                          (uptr *)&REAL(RtlFreeHeap));
+    TryToOverrideFunction("RtlReAllocateHeap", (uptr)WRAP(RtlReAllocateHeap),
+                          (uptr *)&REAL(RtlReAllocateHeap));
+    TryToOverrideFunction("RtlAllocateHeap", (uptr)WRAP(RtlAllocateHeap),
+                          (uptr *)&REAL(RtlAllocateHeap));
   } else {
 #define INTERCEPT_UCRT_FUNCTION(func)                                  \
   if (!INTERCEPT_FUNCTION_DLLIMPORT(                                   \

>From 02765547e3bca72f86af3b6d6e39b0b6c1670c77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bernhard=20=C3=9Cbelacker?= <bernhardu at mailbox.org>
Date: Thu, 16 Oct 2025 20:35:16 +0200
Subject: [PATCH 2/2] fixup! [sanitizers][windows] Add report on intercept fail
 to more Rtl* functions.

---
 compiler-rt/lib/asan/asan_malloc_win.cpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/compiler-rt/lib/asan/asan_malloc_win.cpp b/compiler-rt/lib/asan/asan_malloc_win.cpp
index a122834007eab..9ae266a59e1bc 100644
--- a/compiler-rt/lib/asan/asan_malloc_win.cpp
+++ b/compiler-rt/lib/asan/asan_malloc_win.cpp
@@ -485,8 +485,8 @@ INTERCEPTOR_WINAPI(void*, RtlReAllocateHeap, HANDLE HeapHandle, DWORD Flags,
 
 namespace __asan {
 
-static void TryToOverrideFunction(const char *fname, uptr new_func,
-                                  uptr *orig_old_func = 0) {
+static void TryToOverrideFunction(const char* fname, uptr new_func,
+                                  uptr* orig_old_func = 0) {
   // Failure here is not fatal. The CRT may not be present, and different CRT
   // versions use different symbols.
   if (!__interception::OverrideFunction(fname, new_func, orig_old_func))
@@ -521,13 +521,13 @@ void ReplaceSystemMalloc() {
 
     // Undocumented functions must be intercepted by name, not by symbol.
     TryToOverrideFunction("RtlSizeHeap", (uptr)WRAP(RtlSizeHeap),
-                          (uptr *)&REAL(RtlSizeHeap));
+                          (uptr*)&REAL(RtlSizeHeap));
     TryToOverrideFunction("RtlFreeHeap", (uptr)WRAP(RtlFreeHeap),
-                          (uptr *)&REAL(RtlFreeHeap));
+                          (uptr*)&REAL(RtlFreeHeap));
     TryToOverrideFunction("RtlReAllocateHeap", (uptr)WRAP(RtlReAllocateHeap),
-                          (uptr *)&REAL(RtlReAllocateHeap));
+                          (uptr*)&REAL(RtlReAllocateHeap));
     TryToOverrideFunction("RtlAllocateHeap", (uptr)WRAP(RtlAllocateHeap),
-                          (uptr *)&REAL(RtlAllocateHeap));
+                          (uptr*)&REAL(RtlAllocateHeap));
   } else {
 #define INTERCEPT_UCRT_FUNCTION(func)                                  \
   if (!INTERCEPT_FUNCTION_DLLIMPORT(                                   \