[clang] [clang][CodeGen][MSVC] Return vector types from methods indirectly (PR #157365)

[via cfe-commits]

llvmbot wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-clang-codegen

Author: Henry Baba-Weiss (henrybw)

<details>
<summary>Changes</summary>

The MSVC ABI almost always returns vector types directly, but on x86 and x86_64, there seems to be a special case for member functions, which return vector types indirectly.

Fixes #<!-- -->104.

---
Full diff: https://github.com/llvm/llvm-project/pull/157365.diff


3 Files Affected:

- (modified) clang/docs/ReleaseNotes.rst (+3) 
- (modified) clang/lib/CodeGen/MicrosoftCXXABI.cpp (+14-9) 
- (added) clang/test/CodeGenCXX/microsoft-abi-vector-types.cpp (+36) 


``````````diff
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index d1ef91b7e7c14..1be917221aa2c 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -90,6 +90,9 @@ C++ Specific Potentially Breaking Changes
 ABI Changes in This Version
 ---------------------------
 
+- Fixed Microsoft calling convention for returning vector types from C++ member
+  functions. Such vector types should be returned indirectly. (GH#104)
+
 AST Dumping Potentially Breaking Changes
 ----------------------------------------
 - How nested name specifiers are dumped and printed changes, keeping track of clang AST changes.
diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
index 88f0648660965..2d76eebcecd08 100644
--- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp
+++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
@@ -1168,15 +1168,20 @@ static bool isTrivialForMSVC(const CXXRecordDecl *RD, QualType Ty,
 }
 
 bool MicrosoftCXXABI::classifyReturnType(CGFunctionInfo &FI) const {
-  const CXXRecordDecl *RD = FI.getReturnType()->getAsCXXRecordDecl();
-  if (!RD)
-    return false;
-
-  bool isTrivialForABI = RD->canPassInRegisters() &&
-                         isTrivialForMSVC(RD, FI.getReturnType(), CGM);
-
-  // MSVC always returns structs indirectly from C++ instance methods.
-  bool isIndirectReturn = !isTrivialForABI || FI.isInstanceMethod();
+  bool isIndirectReturn = false;
+  if (const CXXRecordDecl *RD = FI.getReturnType()->getAsCXXRecordDecl()) {
+    bool isTrivialForABI = RD->canPassInRegisters() &&
+                           isTrivialForMSVC(RD, FI.getReturnType(), CGM);
+
+    // MSVC always returns structs indirectly from C++ instance methods.
+    isIndirectReturn = !isTrivialForABI || FI.isInstanceMethod();
+  } else if (isa<VectorType>(FI.getReturnType())) {
+    // On x86, MSVC seems to only return vector types indirectly from non-
+    // vectorcall C++ instance methods.
+    isIndirectReturn =
+        CGM.getTarget().getTriple().isX86() && FI.isInstanceMethod() &&
+        FI.getCallingConvention() != llvm::CallingConv::X86_VectorCall;
+  }
 
   if (isIndirectReturn) {
     CharUnits Align = CGM.getContext().getTypeAlignInChars(FI.getReturnType());
diff --git a/clang/test/CodeGenCXX/microsoft-abi-vector-types.cpp b/clang/test/CodeGenCXX/microsoft-abi-vector-types.cpp
new file mode 100644
index 0000000000000..e046fb4bb3169
--- /dev/null
+++ b/clang/test/CodeGenCXX/microsoft-abi-vector-types.cpp
@@ -0,0 +1,36 @@
+// RUN: %clang_cc1 -ffreestanding -emit-llvm %s -o - -triple=i686-pc-windows-msvc | FileCheck %s
+// RUN: %clang_cc1 -ffreestanding -emit-llvm %s -o - -triple=x86_64-pc-windows-msvc | FileCheck %s
+
+// To match the MSVC ABI, vector types must be returned indirectly from member
+// functions (as long as they do not use the vectorcall calling convention),
+// but must be returned directly everywhere else.
+
+#include <xmmintrin.h>
+
+struct Foo {
+  __m128 method_m128();
+  __m128 __vectorcall vectorcall_method_m128();
+};
+
+__m128 Foo::method_m128() {
+  return __m128{};
+// GH104
+// CHECK: store <4 x float>
+// CHECK: ret void
+}
+
+__m128 __vectorcall Foo::vectorcall_method_m128() {
+  return __m128{};
+// CHECK: ret <4 x float>
+}
+
+__m128 func_m128() {
+  return __m128{};
+// CHECK: ret <4 x float>
+}
+
+__m128 __vectorcall vectorcall_func_m128() {
+  return __m128{};
+// CHECK: ret <4 x float>
+}
+

``````````

</details>


https://github.com/llvm/llvm-project/pull/157365