[compiler-rt] Ensure Symbolize Flag setting on Windows through __asan_default_options() is maintained throughout runtime (PR #132811)

[MacGyver Codilla via llvm-commits]

https://github.com/39otsu created https://github.com/llvm/llvm-project/pull/132811

As a consequence of the ASAN DLL's initialization process on Windows, some flags defined by the user through overriding the __asan_default_options() method will not be honored. More information here: [#117925](https://github.com/llvm/llvm-project/issues/117925)

This PR aims to alleviate this for the symbolize flag in relation to this user's concern [here.](https://developercommunity.visualstudio.com/t/Overloading-of-__asan_default_options-/10688871)

1. Declared `Symbolizer::UpdateSymbolizerTools()`.
2. Defined `Symbolizer::UpdateSymbolizerTools()`. Upon invocation of the weak function callback of `__asan_default_options()`, `Symbolizer::tools_` will be cleared if the user specifies `symbolize=0`.
3. Added tests.

>From 622216fd2e91ae72de1229d04a7c4146b940e784 Mon Sep 17 00:00:00 2001
From: MacGyver Codilla <mcodilla at microsoft.com>
Date: Wed, 19 Mar 2025 20:08:21 +0000
Subject: [PATCH] Implemented Update to Symbolizer Tools list after user
 defined function becomes available on Windows.

---
 compiler-rt/lib/asan/asan_flags.cpp           |  3 +-
 .../sanitizer_common/sanitizer_symbolizer.h   |  3 ++
 .../sanitizer_symbolizer_libcdep.cpp          | 13 +++++++
 .../test/asan/TestCases/Windows/symbolize.cpp | 38 +++++++++++++++++++
 4 files changed, 56 insertions(+), 1 deletion(-)
 create mode 100644 compiler-rt/test/asan/TestCases/Windows/symbolize.cpp

diff --git a/compiler-rt/lib/asan/asan_flags.cpp b/compiler-rt/lib/asan/asan_flags.cpp
index 9cfb70bd00c78..955560d77a683 100644
--- a/compiler-rt/lib/asan/asan_flags.cpp
+++ b/compiler-rt/lib/asan/asan_flags.cpp
@@ -247,7 +247,8 @@ void InitializeFlags() {
         // See GH issue 'https://github.com/llvm/llvm-project/issues/117925' for
         // details.
         SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
-      });
+        Symbolizer::UpdateSymbolizerTools();
+    });
 
 #  if CAN_SANITIZE_UB
   AddRegisterWeakFunctionCallback(
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h
index bd89dc4e302fc..604f143513193 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer.h
@@ -136,6 +136,9 @@ class Symbolizer final {
   /// (if it wasn't already initialized).
   static Symbolizer *GetOrInit();
   static void LateInitialize();
+#if SANITIZER_WINDOWS
+  static void UpdateSymbolizerTools();
+#endif
   // Returns a list of symbolized frames for a given address (containing
   // all inlined functions, if necessary).
   SymbolizedStack *SymbolizePC(uptr address);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cpp
index 74458028ae8f5..320f92746df0c 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libcdep.cpp
@@ -26,6 +26,19 @@ Symbolizer *Symbolizer::GetOrInit() {
   return symbolizer_;
 }
 
+#if SANITIZER_WINDOWS
+// If the 'symbolize' flag is set to 0, it clears the tools
+// associated with the symbolizer to prevent unnecessary symbolization and
+// resource usage. This is necessary because of the late binding of the
+// overridden method, __asan_default_options().
+void Symbolizer::UpdateSymbolizerTools() {
+  SpinMutexLock l(&init_mu_);
+  if (!common_flags()->symbolize) {
+    symbolizer_->tools_.clear();
+  }
+}
+#endif
+
 // See sanitizer_symbolizer_markup.cpp.
 #if !SANITIZER_SYMBOLIZER_MARKUP
 
diff --git a/compiler-rt/test/asan/TestCases/Windows/symbolize.cpp b/compiler-rt/test/asan/TestCases/Windows/symbolize.cpp
new file mode 100644
index 0000000000000..7f8cd5aea633b
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/Windows/symbolize.cpp
@@ -0,0 +1,38 @@
+// RUN: %clangxx_asan -O0 %s -o %t
+// RUN: %env_asan_opts=symbolize=0 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-SYMBOLIZE-OFF
+// RUN: %env_asan_opts=symbolize=1 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-SYMBOLIZE-ON
+
+// RUN: %clangxx_asan -O0 %s -o %t -DUSER_FUNCTION_OFF
+// RUN: not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-SYMBOLIZE-OFF
+// RUN: %env_asan_opts=symbolize=0 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-SYMBOLIZE-OFF
+// RUN: %env_asan_opts=symbolize=1 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-SYMBOLIZE-ON
+
+// RUN: %clangxx_asan -O0 %s -o %t -DUSER_FUNCTION_ON
+// RUN: not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-SYMBOLIZE-ON
+// RUN: %env_asan_opts=symbolize=0 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-SYMBOLIZE-OFF
+// RUN: %env_asan_opts=symbolize=1 not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-SYMBOLIZE-ON
+#if USER_FUNCTION_OFF
+
+extern "C" __declspec(dllexport) extern const char *__asan_default_options() {
+  return "symbolize=0";
+}
+
+#endif
+
+#if USER_FUNCTION_ON
+
+extern "C" __declspec(dllexport) extern const char *__asan_default_options() {
+  return "symbolize=1";
+}
+
+#endif
+
+#include <cstdio>
+#include <cstdlib>
+
+volatile static int heapBufferOverflowValue = 10;
+int main() {
+  int *array = new int[10];
+  heapBufferOverflowValue = array[10]; // CHECK-SYMBOLIZE-ON: symbolize.cpp:36
+  return 0; // CHECK-SYMBOLIZE-OFF: symbolize.cpp.tmp+0x
+}
\ No newline at end of file