[compiler-rt] [sanitizer] Fail __sanitizer_symbolize_demangle instead of returning input (PR #66006)

[Vitaly Buka via llvm-commits]

https://github.com/vitalybuka updated https://github.com/llvm/llvm-project/pull/66006:

>From ba97e80b06cc5d69348afb7e157a1fbd68313788 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka at google.com>
Date: Sat, 9 Sep 2023 13:15:33 -0700
Subject: [PATCH] [sanitizer] Fail __sanitizer_symbolize_demangle instead of
 returning input

LLVMSymbolizer::DemangleName returns the same input if it can't demangle.
We can't tell if this is already demangled or format is unsupported.

Internally DemangleName uses nonMicrosoftDemangle which can report a failure.
---
 .../symbolizer/sanitizer_symbolize.cpp        |  6 +++--
 .../TestCases/demangle_internal.cpp           | 25 +++++++++++++++++++
 .../TestCases/TypeCheck/vptr-virtual-base.cpp |  3 ---
 .../test/ubsan/TestCases/TypeCheck/vptr.cpp   |  3 ---
 4 files changed, 29 insertions(+), 8 deletions(-)
 create mode 100644 compiler-rt/test/sanitizer_common/TestCases/demangle_internal.cpp

diff --git a/compiler-rt/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cpp b/compiler-rt/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cpp
index 6ca4b9835bf3acd..f6dac78337068b5 100644
--- a/compiler-rt/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cpp
+++ b/compiler-rt/lib/sanitizer_common/symbolizer/sanitizer_symbolize.cpp
@@ -17,6 +17,7 @@
 
 #include "llvm/DebugInfo/Symbolize/DIPrinter.h"
 #include "llvm/DebugInfo/Symbolize/Symbolize.h"
+#include "llvm/Demangle/Demangle.h"
 
 static llvm::symbolize::LLVMSymbolizer *Symbolizer = nullptr;
 static bool Demangle = true;
@@ -117,8 +118,9 @@ void __sanitizer_symbolize_flush() {
 
 bool __sanitizer_symbolize_demangle(const char *Name, char *Buffer,
                                    int MaxLength) {
-  std::string Result =
-      llvm::symbolize::LLVMSymbolizer::DemangleName(Name, nullptr);
+  std::string Result;
+  if (!llvm::nonMicrosoftDemangle(Name, Result))
+    return false;
   return __sanitizer::internal_snprintf(Buffer, MaxLength, "%s",
                                         Result.c_str()) < MaxLength;
 }
diff --git a/compiler-rt/test/sanitizer_common/TestCases/demangle_internal.cpp b/compiler-rt/test/sanitizer_common/TestCases/demangle_internal.cpp
new file mode 100644
index 000000000000000..3202fc17aea7d63
--- /dev/null
+++ b/compiler-rt/test/sanitizer_common/TestCases/demangle_internal.cpp
@@ -0,0 +1,25 @@
+// RUN: %clangxx -O0 %s -o %t && %run %t
+
+// REQUIRES: internal_symbolizer
+
+// FIXME: link internal_symbolizer.
+// XFAIL: asan, hwasan, ubsan
+
+#include <algorithm>
+#include <assert.h>
+#include <string.h>
+
+extern "C" bool __sanitizer_symbolize_demangle(const char *Name, char *Buffer,
+                                               int MaxLength);
+
+int main() {
+  char out[128];
+  assert(!__sanitizer_symbolize_demangle("1A", out, sizeof(out)));
+
+  const char name[] = "_Z3fooi";
+  for (int i = 1; i < sizeof(out); ++i) {
+    memset(out, 1, sizeof(out));
+    assert(__sanitizer_symbolize_demangle(name, out, i) == (i > 8));
+    assert(i < 9 || 0 == strncmp(out, "foo(int)", i - 1));
+  }
+}
diff --git a/compiler-rt/test/ubsan/TestCases/TypeCheck/vptr-virtual-base.cpp b/compiler-rt/test/ubsan/TestCases/TypeCheck/vptr-virtual-base.cpp
index 7b6847a9ad6bec1..9d44f0230e92452 100644
--- a/compiler-rt/test/ubsan/TestCases/TypeCheck/vptr-virtual-base.cpp
+++ b/compiler-rt/test/ubsan/TestCases/TypeCheck/vptr-virtual-base.cpp
@@ -1,9 +1,6 @@
 // RUN: %clangxx -frtti -fsanitize=null,vptr -fno-sanitize-recover=vptr -g %s -O3 -o %t
 // RUN: not %run %t 2>&1 | FileCheck %s
 
-// FIXME: Investigate.
-// XFAIL: internal_symbolizer && (ubsan-tsan || ubsan-msan)
-
 // REQUIRES: shared_cxxabi
 // REQUIRES: cxxabi
 // UNSUPPORTED: target={{.*windows-msvc.*}}
diff --git a/compiler-rt/test/ubsan/TestCases/TypeCheck/vptr.cpp b/compiler-rt/test/ubsan/TestCases/TypeCheck/vptr.cpp
index 3c3af9a3a340cfe..2d0e48cd84f3cff 100644
--- a/compiler-rt/test/ubsan/TestCases/TypeCheck/vptr.cpp
+++ b/compiler-rt/test/ubsan/TestCases/TypeCheck/vptr.cpp
@@ -36,9 +36,6 @@
 // RUN: echo "vptr_check:S" > %t.loc-supp
 // RUN: %env_ubsan_opts=halt_on_error=1:suppressions='"%t.loc-supp"' not %run %t x- 2>&1 | FileCheck %s --check-prefix=CHECK-LOC-SUPPRESS
 
-// FIXME: Investigate.
-// XFAIL: internal_symbolizer && (ubsan-tsan || ubsan-msan)
-
 // REQUIRES: stable-runtime, cxxabi
 // UNSUPPORTED: target={{.*windows-msvc.*}}
 // Suppressions file not pushed to the device.