[PATCH] Tighten diagnostics for calling conventions on variadic functions

[Hans Wennborg]

Hi rnk,

This is a follow-up to r192240.

We shouldn't allow __thiscall on variadic functions; that's just too weird, and we currently assert in codegen if we try.

MSVC silently ignores __stdcall and __fastcall on variadic functions, and so does GCC. This patch makes us do the same, but with a warning.

http://llvm-reviews.chandlerc.com/D1870

Files:
  lib/Sema/SemaType.cpp
  test/CodeGen/microsoft-call-conv.c
  test/Sema/callingconv.c
  test/Sema/mrtd.c
  test/SemaCXX/calling-conv-compat.cpp

Index: lib/Sema/SemaType.cpp
===================================================================
--- lib/Sema/SemaType.cpp
+++ lib/Sema/SemaType.cpp
@@ -4542,22 +4542,24 @@
     }
   }
 
-  // Diagnose the use of X86 fastcall on varargs or unprototyped functions.
-  if (CC == CC_X86FastCall) {
-    if (isa<FunctionNoProtoType>(fn)) {
-      S.Diag(attr.getLoc(), diag::err_cconv_knr)
-        << FunctionType::getNameForCallConv(CC);
+  // Diagnose use of X86 stdcall/fastcall/thiscall on variadic functions.
+  if (CC == CC_X86StdCall || CC == CC_X86FastCall || CC == CC_X86ThisCall) {
+    const FunctionProtoType *FnP = dyn_cast<FunctionProtoType>(fn);
+    if (FnP && FnP->isVariadic()) {
+      // We don't support thiscall on variadic functions. stdcall and fastcall
+      // are ignored with a warning for GCC and MS compatibility.
+      unsigned DiagID = CC == CC_X86ThisCall ? diag::err_cconv_varargs
+                                             : diag::warn_cconv_varargs;
+      S.Diag(attr.getLoc(), DiagID) << FunctionType::getNameForCallConv(CC);
       attr.setInvalid();
       return true;
     }
+  }
 
-    const FunctionProtoType *FnP = cast<FunctionProtoType>(fn);
-    if (FnP->isVariadic()) {
-      // In MS compatibility mode, this is just a warning.
-      const LangOptions &L = S.getLangOpts();
-      unsigned DiagID = L.MicrosoftMode ? diag::warn_cconv_varargs
-                                        : diag::err_cconv_varargs;
-      S.Diag(attr.getLoc(), DiagID)
+  // Diagnose the use of X86 fastcall on unprototyped functions.
+  if (CC == CC_X86FastCall) {
+    if (isa<FunctionNoProtoType>(fn)) {
+      S.Diag(attr.getLoc(), diag::err_cconv_knr)
         << FunctionType::getNameForCallConv(CC);
       attr.setInvalid();
       return true;
Index: test/CodeGen/microsoft-call-conv.c
===================================================================
--- test/CodeGen/microsoft-call-conv.c
+++ test/CodeGen/microsoft-call-conv.c
@@ -1,5 +1,6 @@
 // RUN: %clang_cc1 -triple i386-pc-linux -emit-llvm < %s | FileCheck %s
-// RUN: %clang_cc1 -triple i386-pc-linux -emit-llvm -fms-compatibility -DWIN < %s | FileCheck --check-prefix=WIN %s
+// RUN: %clang_cc1 -triple i386-pc-linux -emit-llvm -mrtd < %s | FileCheck %s
+// RUN: %clang_cc1 -triple i386-pc-linux -emit-llvm -fms-compatibility < %s
 
 void __fastcall f1(void);
 void __stdcall f2(void);
@@ -51,9 +52,9 @@
 }
 
 // PR12535
-#ifdef WIN
 void __fastcall f9(int x, int y) {};
 // WIN: define x86_fastcallcc void @f9({{.*}})
 void __fastcall f10(int x, ...) {};
 // WIN: define void @f10({{.*}})
-#endif
+void __stdcall f11(int x, ...) {};
+// WIN: define void @f11({{.*}})
Index: test/Sema/callingconv.c
===================================================================
--- test/Sema/callingconv.c
+++ test/Sema/callingconv.c
@@ -16,13 +16,12 @@
 void __attribute__((fastcall)) test1(void) {
 }
 
-#ifdef WIN
 void __attribute__((fastcall)) test2(int a, ...) { // expected-warning {{fastcall calling convention ignored on variadic function}}
 }
-#else
-void __attribute__((fastcall)) test2(int a, ...) { // expected-error {{variadic function cannot use fastcall calling convention}}
+void __attribute__((stdcall)) test3(int a, ...) { // expected-warning {{stdcall calling convention ignored on variadic function}}
+}
+void __attribute__((thiscall)) test4(int a, ...) { // expected-error {{variadic function cannot use thiscall calling convention}}
 }
-#endif
 
 void __attribute__((cdecl)) ctest0() {}
 
Index: test/Sema/mrtd.c
===================================================================
--- test/Sema/mrtd.c
+++ test/Sema/mrtd.c
@@ -12,8 +12,7 @@
 void nonvariadic2(int a, int b, int c);
 void __attribute__((stdcall)) nonvariadic2(int a, int b, int c) { }
 
-// expected-note at +2 {{previous declaration is here}}
-// expected-error at +2 {{function declared 'stdcall' here was previously declared without calling convention}}
+// expected-warning at +2 {{stdcall calling convention ignored on variadic function}}
 void variadic(int a, ...);
 void __attribute__((stdcall)) variadic(int a, ...);
 
@@ -34,7 +33,6 @@
 extern void (*c)(int, int);
 __attribute__((stdcall)) extern void (*c)(int, int);
 
-// expected-note at +2 {{previous definition is here}}
-// expected-error at +2 {{redefinition of 'd' with a different type: 'void ((*))(int, ...) __attribute__((stdcall))' vs 'void (*)(int, ...)'}}
+// expected-warning at +2 {{stdcall calling convention ignored on variadic function}}
 extern void (*d)(int, ...);
 __attribute__((stdcall)) extern void (*d)(int, ...);
Index: test/SemaCXX/calling-conv-compat.cpp
===================================================================
--- test/SemaCXX/calling-conv-compat.cpp
+++ test/SemaCXX/calling-conv-compat.cpp
@@ -248,7 +248,7 @@
 struct A {
   void            member_default(int, ...);
   void __cdecl    member_cdecl(int, ...);
-  void __thiscall member_thiscall(int, ...);
+  void __thiscall member_thiscall(int, ...); // expected-error {{variadic function cannot use thiscall calling convention}}
 };
 
 struct B : public A {
-------------- next part --------------
A non-text attachment was scrubbed...
Name: D1870.1.patch
Type: text/x-patch
Size: 5078 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/cfe-commits/attachments/20131009/cd85c7ad/attachment.bin>