[PATCH] [ms-cxxabi] Mangle function pointer arguments correctly

[Reid Kleckner]

Hi rsmith, pcc,

First, MSVC seems to model decayed array parameters as const pointers
instead of plain pointers.  This isn't really observable in any way
other than through mangling, so only do it for C++ and for the Microsoft
C++ ABI.

Second, avoid type canonicalization in MicrosoftMangle::mangleType(),
since that strips away qualifiers on function pointer arguments, which
MSVC includes in the mangling.

Fixes PR16096.

http://llvm-reviews.chandlerc.com/D844

Files:
  lib/AST/ASTContext.cpp
  lib/AST/MicrosoftMangle.cpp
  test/CodeGenCXX/mangle-ms-arg-qualifiers.cpp

Index: lib/AST/ASTContext.cpp
===================================================================
--- lib/AST/ASTContext.cpp
+++ lib/AST/ASTContext.cpp
@@ -4182,7 +4182,12 @@
   QualType PtrTy = getPointerType(PrettyArrayType->getElementType());
 
   // int x[restrict 4] ->  int *restrict
-  return getQualifiedType(PtrTy, PrettyArrayType->getIndexTypeQualifiers());
+  Qualifiers qs = PrettyArrayType->getIndexTypeQualifiers();
+  // MSVC seems to say that 'int x[4]' decays to 'int *const', which really only
+  // matters for C++ name mangling.
+  if (getTargetInfo().getCXXABI().isMicrosoft())
+    qs.addConst();
+  return getQualifiedType(PtrTy, qs);
 }
 
 QualType ASTContext::getBaseElementType(const ArrayType *array) const {
Index: lib/AST/MicrosoftMangle.cpp
===================================================================
--- lib/AST/MicrosoftMangle.cpp
+++ lib/AST/MicrosoftMangle.cpp
@@ -126,7 +126,7 @@
   void mangleFunctionType(const FunctionType *T, const FunctionDecl *D,
                           bool IsStructor, bool IsInstMethod);
   void mangleDecayedArrayType(const ArrayType *T, bool IsGlobal);
-  void mangleArrayType(const ArrayType *T, Qualifiers Quals);
+  void mangleArrayType(const ArrayType *T);
   void mangleFunctionClass(const FunctionDecl *FD);
   void mangleCallingConvention(const FunctionType *T, bool IsInstMethod = false);
   void mangleIntegerLiteral(const llvm::APSInt &Number, bool IsBoolean);
@@ -978,6 +978,8 @@
 
 void MicrosoftCXXNameMangler::mangleArgumentType(QualType T,
                                                  SourceRange Range) {
+  // MSVC will backreference two canonically equivalent types that have slightly
+  // different manglings when mangled alone.
   void *TypePtr = getASTContext().getCanonicalType(T).getAsOpaquePtr();
   ArgBackRefMap::iterator Found = TypeBackReferences.find(TypePtr);
 
@@ -1008,16 +1010,19 @@
 
 void MicrosoftCXXNameMangler::mangleType(QualType T, SourceRange Range,
                                          QualifierMangleMode QMM) {
-  // Only operate on the canonical type!
-  T = getASTContext().getCanonicalType(T);
+  // Don't use the canonical types.  MSVC includes things like 'const' on
+  // pointer arguments to function pointers that canonicalization strips away.
+  T = T.getDesugaredType(getASTContext());
   Qualifiers Quals = T.getLocalQualifiers();
 
-  if (const ArrayType *AT = dyn_cast<ArrayType>(T)) {
+  if (const ArrayType *AT = getASTContext().getAsArrayType(T)) {
+    // If there were any Quals, getAsArrayType() pushed them onto the array
+    // element type.
     if (QMM == QMM_Mangle)
       Out << 'A';
     else if (QMM == QMM_Escape || QMM == QMM_Result)
       Out << "$$B";
-    mangleArrayType(AT, Quals);
+    mangleArrayType(AT);
     return;
   }
 
@@ -1429,8 +1434,7 @@
                                          SourceRange) {
   llvm_unreachable("Should have been special cased");
 }
-void MicrosoftCXXNameMangler::mangleArrayType(const ArrayType *T,
-                                              Qualifiers Quals) {
+void MicrosoftCXXNameMangler::mangleArrayType(const ArrayType *T) {
   QualType ElementTy(T, 0);
   SmallVector<llvm::APInt, 3> Dimensions;
   for (;;) {
@@ -1469,8 +1473,7 @@
   mangleNumber(Dimensions.size());
   for (unsigned Dim = 0; Dim < Dimensions.size(); ++Dim)
     mangleNumber(Dimensions[Dim].getLimitedValue());
-  mangleType(getASTContext().getQualifiedType(ElementTy.getTypePtr(), Quals),
-             SourceRange(), QMM_Escape);
+  mangleType(ElementTy, SourceRange(), QMM_Escape);
 }
 
 // <type>                   ::= <pointer-to-member-type>
Index: test/CodeGenCXX/mangle-ms-arg-qualifiers.cpp
===================================================================
--- test/CodeGenCXX/mangle-ms-arg-qualifiers.cpp
+++ test/CodeGenCXX/mangle-ms-arg-qualifiers.cpp
@@ -162,3 +162,34 @@
 void foo(Vector*, const Vector, const double) {}
 // CHECK: "\01?foo@@YAXPAY02NQBNN at Z"
 // X64: "\01?foo@@YAXPEAY02NQEBNN at Z"
+
+typedef void (*ConstFunPtr)(int *const d);
+void foo_fnptrconst(ConstFunPtr f) {  }
+// CHECK: "\01?foo_fnptrconst@@YAXP6AXQAH at Z@Z"
+// X64:   "\01?foo_fnptrconst@@YAXP6AXQEAH at Z@Z"
+
+typedef void (*ArrayFunPtr)(int d[1]);
+void foo_fnptrarray(ArrayFunPtr f) {  }
+// CHECK: "\01?foo_fnptrarray@@YAXP6AXQAH at Z@Z"
+// X64:   "\01?foo_fnptrarray@@YAXP6AXQEAH at Z@Z"
+
+void foo_fnptrbackref1(ArrayFunPtr f1, ArrayFunPtr f2) {  }
+// CHECK: "\01?foo_fnptrbackref1@@YAXP6AXQAH at Z1@Z"
+// X64:   "\01?foo_fnptrbackref1@@YAXP6AXQEAH at Z1@Z"
+
+void foo_fnptrbackref2(ArrayFunPtr f1, ConstFunPtr f2) {  }
+// CHECK: "\01?foo_fnptrbackref2@@YAXP6AXQAH at Z1@Z"
+// X64:   "\01?foo_fnptrbackref2@@YAXP6AXQEAH at Z1@Z"
+
+typedef void (*NormalFunPtr)(int *d);
+void foo_fnptrbackref3(ArrayFunPtr f1, NormalFunPtr f2) {  }
+// CHECK: "\01?foo_fnptrbackref3@@YAXP6AXQAH at Z1@Z"
+// X64:   "\01?foo_fnptrbackref3@@YAXP6AXQEAH at Z1@Z"
+
+void foo_fnptrbackref4(NormalFunPtr f1, ArrayFunPtr f2) {  }
+// CHECK: "\01?foo_fnptrbackref4@@YAXP6AXPAH at Z1@Z"
+// X64:   "\01?foo_fnptrbackref4@@YAXP6AXPEAH at Z1@Z"
+
+ArrayFunPtr ret_fnptrarray() { return 0; }
+// CHECK: "\01?ret_fnptrarray@@YAP6AXQAH at ZXZ"
+// X64:   "\01?ret_fnptrarray@@YAP6AXQEAH at ZXZ"
-------------- next part --------------
A non-text attachment was scrubbed...
Name: D844.1.patch
Type: text/x-patch
Size: 5210 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/cfe-commits/attachments/20130522/42084110/attachment.bin>