[llvm] [MSVC][Demangling] Respect MSDF_NoCallingConvention for function pointers (PR #209113)

[Fred Tingaud via llvm-commits]

https://github.com/frederic-tingaud-sonarsource updated https://github.com/llvm/llvm-project/pull/209113

>From 01a1af2571e9c958c46cdca1a516d5a0f99697ee Mon Sep 17 00:00:00 2001
From: Fred Tingaud <frederic.tingaud at sonarsource.com>
Date: Wed, 10 Jun 2026 11:52:26 +0200
Subject: [PATCH] [MSVC][Demangling] Respect MSDF_NoCallingConvention for
 function pointers

Function pointers used to use a trick where it would drop all the flags and
pass OF_NoCallingConvention to put the calling convention inside the
parentheses. In order to respect MSDF_NoCallingConvention, we need to get
rid of this trick.

A side effect of the trick was that OF_NoReturnType was also lost, which is
actually good because dropping the return type of a named function is
unambiguous, but dropping the return type of a function pointer loses
important information. The fix enforces that behavior explicitely.

This change can have impact for end-users who always passed
MSDF_NoCallingConvention, but I believe that this new behavior is the one
they were expecting.

Assisted-by: Claude
---
 .../llvm/Demangle/MicrosoftDemangleNodes.h    |  2 ++
 llvm/lib/Demangle/MicrosoftDemangleNodes.cpp  | 26 +++++++++++++----
 .../Demangle/MicrosoftDemangleTest.cpp        | 29 +++++++++++++++++++
 3 files changed, 51 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h b/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h
index 9a3ebf6a00e59..ce67425143e8c 100644
--- a/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h
+++ b/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h
@@ -356,6 +356,8 @@ struct DEMANGLE_ABI FunctionSignatureNode : public TypeNode {
   void outputPre(OutputBuffer &OB, OutputFlags Flags) const override;
   void outputPost(OutputBuffer &OB, OutputFlags Flags) const override;
 
+  void outputPreSignature(OutputBuffer &OB, OutputFlags Flags) const;
+
   static bool classof(const Node *N) {
     return N->kind() >= NodeKind::FunctionSignature &&
            N->kind() <= NodeKind::FunctionSignatureEnd;
diff --git a/llvm/lib/Demangle/MicrosoftDemangleNodes.cpp b/llvm/lib/Demangle/MicrosoftDemangleNodes.cpp
index daca80b144de5..6efd39a736f7a 100644
--- a/llvm/lib/Demangle/MicrosoftDemangleNodes.cpp
+++ b/llvm/lib/Demangle/MicrosoftDemangleNodes.cpp
@@ -384,8 +384,8 @@ void LiteralOperatorIdentifierNode::output(OutputBuffer &OB,
   outputTemplateParameters(OB, Flags);
 }
 
-void FunctionSignatureNode::outputPre(OutputBuffer &OB,
-                                      OutputFlags Flags) const {
+void FunctionSignatureNode::outputPreSignature(OutputBuffer &OB,
+                                               OutputFlags Flags) const {
   if (!(Flags & OF_NoAccessSpecifier)) {
     if (FunctionClass & FC_Public)
       OB << "public: ";
@@ -411,6 +411,11 @@ void FunctionSignatureNode::outputPre(OutputBuffer &OB,
     ReturnType->outputPre(OB, Flags);
     OB << " ";
   }
+}
+
+void FunctionSignatureNode::outputPre(OutputBuffer &OB,
+                                      OutputFlags Flags) const {
+  outputPreSignature(OB, Flags);
 
   if (!(Flags & OF_NoCallingConvention))
     outputCallingConvention(OB, CallConvention);
@@ -483,7 +488,9 @@ void PointerTypeNode::outputPre(OutputBuffer &OB, OutputFlags Flags) const {
     // It needs to go inside the parentheses.
     const FunctionSignatureNode *Sig =
         static_cast<const FunctionSignatureNode *>(Pointee);
-    Sig->outputPre(OB, OF_NoCallingConvention);
+    // A function pointer's return type is part of its type, so we ignore
+    // OF_NoReturnType.
+    Sig->outputPreSignature(OB, OutputFlags(Flags & ~OF_NoReturnType));
   } else
     Pointee->outputPre(OB, Flags);
 
@@ -498,8 +505,10 @@ void PointerTypeNode::outputPre(OutputBuffer &OB, OutputFlags Flags) const {
     OB << "(";
     const FunctionSignatureNode *Sig =
         static_cast<const FunctionSignatureNode *>(Pointee);
-    outputCallingConvention(OB, Sig->CallConvention);
-    OB << " ";
+    if (!(Flags & OF_NoCallingConvention)) {
+      outputCallingConvention(OB, Sig->CallConvention);
+      OB << " ";
+    }
   }
 
   if (ClassParent) {
@@ -531,7 +540,12 @@ void PointerTypeNode::outputPost(OutputBuffer &OB, OutputFlags Flags) const {
       Pointee->kind() == NodeKind::FunctionSignature)
     OB << ")";
 
-  Pointee->outputPost(OB, Flags);
+  if (Pointee->kind() == NodeKind::FunctionSignature)
+    // A function pointer's return type is part of its type, so we ignore
+    // OF_NoReturnType.
+    Pointee->outputPost(OB, OutputFlags(Flags & ~OF_NoReturnType));
+  else
+    Pointee->outputPost(OB, Flags);
 }
 
 void TagTypeNode::outputPre(OutputBuffer &OB, OutputFlags Flags) const {
diff --git a/llvm/unittests/Demangle/MicrosoftDemangleTest.cpp b/llvm/unittests/Demangle/MicrosoftDemangleTest.cpp
index d45f455a41fb9..902d0bb62711e 100644
--- a/llvm/unittests/Demangle/MicrosoftDemangleTest.cpp
+++ b/llvm/unittests/Demangle/MicrosoftDemangleTest.cpp
@@ -291,10 +291,39 @@ TEST(MicrosoftDemangle, demangleIndirectVariables) {
 TEST(MicrosoftDemangle, demangleFunctionPointers) {
   EXPECT_EQ(microsoftDemangleString("?funcPtr@@3P6AXXZA", MSDF_None),
             "void (__cdecl *funcPtr)(void)");
+  EXPECT_EQ(
+      microsoftDemangleString("?funcPtr@@3P6AXXZA", MSDF_NoCallingConvention),
+      "void (*funcPtr)(void)");
   EXPECT_EQ(microsoftDemangleString("?funcPtr@@3P6AXXZA", MSDF_NoVariableType),
             "funcPtr");
   EXPECT_EQ(microsoftDemangleString("?funcPtr@@3P6AHXZA", MSDF_NoVoidParameter),
             "int (__cdecl *funcPtr)()");
+  // A function pointer's return type is part of its type. Therefore
+  // MSDF_NoReturnType leaves it untouched.
+  EXPECT_EQ(microsoftDemangleString("?funcPtr@@3P6AHXZA", MSDF_NoReturnType),
+            "int (__cdecl *funcPtr)(void)");
+}
+
+TEST(MicrosoftDemangle, demangleNestedMemberPointers) {
+  EXPECT_EQ(
+      microsoftDemangleString("?nestedMemberPtr@@3R8B@@EAAP6AHXZXZEQ1@",
+                              MSDF_None),
+      "int (__cdecl * (__cdecl B::*volatile nestedMemberPtr)(void))(void)");
+  EXPECT_EQ(microsoftDemangleString("?nestedMemberPtr@@3R8B@@EAAP6AHXZXZEQ1@",
+                                    MSDF_NoCallingConvention),
+            "int (* (B::*volatile nestedMemberPtr)(void))(void)");
+  EXPECT_EQ(microsoftDemangleString("?nestedMemberPtr@@3R8B@@EAAP6AHXZXZEQ1@",
+                                    MSDF_NoVariableType),
+            "nestedMemberPtr");
+  EXPECT_EQ(microsoftDemangleString("?nestedMemberPtr@@3R8B@@EAAP6AHXZXZEQ1@",
+                                    MSDF_NoVoidParameter),
+            "int (__cdecl * (__cdecl B::*volatile nestedMemberPtr)())()");
+  // A function pointer's return type is part of its type. Therefore
+  // MSDF_NoReturnType leaves it untouched.
+  EXPECT_EQ(
+      microsoftDemangleString("?nestedMemberPtr@@3R8B@@EAAP6AHXZXZEQ1@",
+                              MSDF_NoReturnType),
+      "int (__cdecl * (__cdecl B::*volatile nestedMemberPtr)(void))(void)");
 }
 
 TEST(MicrosoftDemangle, demangleNestedClass) {