[clang] Suppress returning larger CXX records in mem on PlayStation (PR #161732)

[via cfe-commits]

https://github.com/wjristow created https://github.com/llvm/llvm-project/pull/161732

In commit e8a486ea9789, a change was made so that certain 256-bit and 512-bit CXX records would be returned in memory, fixing a violation of the x86-64 psABI (where they had been incorrectly returned in AVX registers). For compatibility reasons, we want to suppress that ABI-fix on PlayStation. This commit suppresses that change for PlayStation, and updates the test to include checking the 512-bit case.

>From 736f72af93fc7515032a4023984768754c0c4234 Mon Sep 17 00:00:00 2001
From: Warren Ristow <warren.ristow at sony.com>
Date: Thu, 2 Oct 2025 13:29:53 -0700
Subject: [PATCH] Suppress returning larger CXX records in mem on PlayStation

In commit e8a486ea9789, a change was made so that certain 256-bit and
512-bit CXX records would be returned in memory, fixing a violation of
the x86-64 psABI (where they had been incorrectly returned in AVX
registers). For compatibility reasons, we want to suppress that ABI-fix
on PlayStation. This commit suppresses that change for PlayStation, and
updates the test to include checking the 512-bit case.
---
 clang/lib/CodeGen/Targets/X86.cpp         |  5 +++--
 clang/test/CodeGen/X86/avx-cxx-record.cpp | 23 +++++++++++++++++++++++
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index c03ba9487a6dc..fb789489664df 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -1343,9 +1343,10 @@ class X86_64ABIInfo : public ABIInfo {
   }
 
   bool returnCXXRecordGreaterThan128InMem() const {
-    // Clang <= 20.0 did not do this.
+    // Clang <= 20.0 did not do this, and PlayStation does not do this.
     if (getContext().getLangOpts().getClangABICompat() <=
-        LangOptions::ClangABI::Ver20)
+            LangOptions::ClangABI::Ver20 ||
+        getTarget().getTriple().isPS())
       return false;
 
     return true;
diff --git a/clang/test/CodeGen/X86/avx-cxx-record.cpp b/clang/test/CodeGen/X86/avx-cxx-record.cpp
index 6ce6815a521a1..b20bcdd616a43 100644
--- a/clang/test/CodeGen/X86/avx-cxx-record.cpp
+++ b/clang/test/CodeGen/X86/avx-cxx-record.cpp
@@ -1,7 +1,9 @@
 // RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -emit-llvm -O2 -target-cpu x86-64-v3 -o - | FileCheck %s
 // RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -emit-llvm -O2 -target-cpu x86-64-v3 -fclang-abi-compat=20 -o - | FileCheck --check-prefix CLANG-20 %s
+// RUN: %clang_cc1 %s -triple x86_64-sie-ps4 -emit-llvm -O2 -target-cpu x86-64-v3 -o - | FileCheck --check-prefix CLANG-20 %s
 
 using UInt64x2 = unsigned long long __attribute__((__vector_size__(16), may_alias));
+using UInt64x4 = unsigned long long __attribute__((__vector_size__(32), may_alias));
 
 template<int id>
 struct XMM1 {
@@ -23,3 +25,24 @@ XMM2 foo() {
   ((XMM1<1>*)&result)->x = UInt64x2{3, 4};
   return result;
 }
+
+template<int id>
+struct YMM1 {
+    UInt64x4 x;
+};
+
+struct YMM2 : YMM1<0>, YMM1<1> {
+};
+
+// CHECK: define{{.*}} @_Z3barv({{.*}} [[ARG:%.*]]){{.*}}
+// CLANG-20: define{{.*}} <8 x double> @_Z3barv()
+// CHECK: entry:
+// CHECK-NEXT: store {{.*}}, ptr [[ARG]]{{.*}}
+// CHECK-NEXT: [[TMP1:%.*]] = getelementptr {{.*}}, ptr [[ARG]]{{.*}}
+// CHECK-NEXT: store {{.*}}, ptr [[TMP1]]{{.*}}
+YMM2 bar() {
+  YMM2 result;
+  ((YMM1<0>*)&result)->x = UInt64x4{1, 2, 3, 4};
+  ((YMM1<1>*)&result)->x = UInt64x4{5, 6, 7, 8};
+  return result;
+}