[compiler-rt] [compiler-rt][ubsan][nfc-ish] Fix a type conversion bug (PR #100665)

[Alan Zhao via llvm-commits]

https://github.com/alanzhao1 updated https://github.com/llvm/llvm-project/pull/100665

>From b2adefcb692b950616b42939d2dc69a47a3355f9 Mon Sep 17 00:00:00 2001
From: Alan Zhao <ayzhao at google.com>
Date: Thu, 25 Jul 2024 15:36:37 -0700
Subject: [PATCH 1/3] [compiler-rt][ubsan][nfc-ish] Fix a type conversion bug

With https://github.com/llvm/llvm-project/pull/100483, if the inline asm
version of `ptrauth_strip` is used instead of the builtin, the inline
asm implementation will return an unsigned long, causing an incompatible
pointer conversion issue.
---
 compiler-rt/lib/ubsan/ubsan_type_hash_itanium.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/compiler-rt/lib/ubsan/ubsan_type_hash_itanium.cpp b/compiler-rt/lib/ubsan/ubsan_type_hash_itanium.cpp
index 15788574dd995..7cc57268d40da 100644
--- a/compiler-rt/lib/ubsan/ubsan_type_hash_itanium.cpp
+++ b/compiler-rt/lib/ubsan/ubsan_type_hash_itanium.cpp
@@ -207,7 +207,8 @@ struct VtablePrefix {
   std::type_info *TypeInfo;
 };
 VtablePrefix *getVtablePrefix(void *Vtable) {
-  Vtable = ptrauth_strip(Vtable, ptrauth_key_cxx_vtable_pointer);
+  Vtable = reinterpret_cast<void *>(
+      ptrauth_strip(Vtable, ptrauth_key_cxx_vtable_pointer));
   VtablePrefix *Vptr = reinterpret_cast<VtablePrefix*>(Vtable);
   VtablePrefix *Prefix = Vptr - 1;
   if (!IsAccessibleMemoryRange((uptr)Prefix, sizeof(VtablePrefix)))

>From 1deff7995be053b185bfac3b85ff4a8181f81935 Mon Sep 17 00:00:00 2001
From: Alan Zhao <ayzhao at google.com>
Date: Thu, 25 Jul 2024 15:36:37 -0700
Subject: [PATCH 2/3] make `ptrauth_strip` return the same type as value

---
 compiler-rt/lib/sanitizer_common/sanitizer_ptrauth.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_ptrauth.h b/compiler-rt/lib/sanitizer_common/sanitizer_ptrauth.h
index b5215c0d49c06..c9b80c4a11cec 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_ptrauth.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_ptrauth.h
@@ -27,7 +27,7 @@
           : "=r"(ret)                   \
           : "r"(__value)                \
           : "x30");                     \
-      ret;                              \
+      __typeof(__value) ret;            \
     })
 #  define ptrauth_auth_data(__value, __old_key, __old_data) __value
 #  define ptrauth_string_discriminator(__string) ((int)0)

>From 1e01ba0b741903e7e447d69d075a21adfdb89715 Mon Sep 17 00:00:00 2001
From: Alan Zhao <ayzhao at google.com>
Date: Thu, 25 Jul 2024 16:07:02 -0700
Subject: [PATCH 3/3] fix ptrauth_strip fix

---
 compiler-rt/lib/sanitizer_common/sanitizer_ptrauth.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_ptrauth.h b/compiler-rt/lib/sanitizer_common/sanitizer_ptrauth.h
index c9b80c4a11cec..265a9925a15a0 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_ptrauth.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_ptrauth.h
@@ -18,7 +18,7 @@
 // the NOP space so will do nothing when it is not enabled or not available.
 #  define ptrauth_strip(__value, __key) \
     ({                                  \
-      unsigned long ret;                \
+      __typeof(__value) ret;            \
       asm volatile(                     \
           "mov x30, %1\n\t"             \
           "hint #7\n\t"                 \
@@ -27,7 +27,7 @@
           : "=r"(ret)                   \
           : "r"(__value)                \
           : "x30");                     \
-      __typeof(__value) ret;            \
+      ret;                              \
     })
 #  define ptrauth_auth_data(__value, __old_key, __old_data) __value
 #  define ptrauth_string_discriminator(__string) ((int)0)