[clang] [clang] [x86] Rewrite the Sys V Classification Algorithm to Better Match SysV ABI and gcc (PR #221746)

Benjamin Luke via cfe-commits cfe-commits at lists.llvm.org
Tue Nov 3 16:46:51 PST 2026


https://github.com/freaknbigpanda updated https://github.com/llvm/llvm-project/pull/221746

>From 85177f987743b27d9086fea9b6a107b1fd7e90d0 Mon Sep 17 00:00:00 2001
From: Benjamin Luke <benjamin.luke at sony.com>
Date: Fri, 20 Mar 2026 15:21:45 -0700
Subject: [PATCH 01/18] [clang][X86] Refactor X86_64AbiInfo::classify to
 properly implement psABI sepc for types >128bits

todo: fill out commit message later
---
 clang/include/clang/Basic/ABIVersions.def     |   5 +
 clang/lib/CodeGen/Targets/X86.cpp             | 821 +++++++++++++++---
 clang/test/CodeGen/X86/x86-illegal-vector.c   |  22 +
 .../CodeGen/X86/x86-small-fp-vector-abi.c     |  58 ++
 clang/test/CodeGen/X86/x86-vec-i128.c         |  28 +-
 .../CodeGen/X86/x86_64-atomic-vector-abi.c    |  92 ++
 clang/test/CodeGen/builtin-masked.c           |  92 +-
 .../CodeGen/builtin-maximumnum-minimumnum.c   |  18 +-
 clang/test/CodeGen/builtin-maxnum-minnum.c    |  18 +-
 clang/test/CodeGen/nofpclass.c                | 514 +++++------
 clang/test/CodeGen/sysv_abi.c                 |  28 +-
 clang/test/CodeGenCXX/ext-int-vector-abi.cpp  |   8 +-
 .../test/CodeGenCXX/matrix-vector-bit-int.cpp |  20 +-
 .../x86_64-empty-base-vector-abi.cpp          |  51 ++
 14 files changed, 1268 insertions(+), 507 deletions(-)
 create mode 100644 clang/test/CodeGen/X86/x86-illegal-vector.c
 create mode 100644 clang/test/CodeGen/X86/x86-small-fp-vector-abi.c
 create mode 100644 clang/test/CodeGen/X86/x86_64-atomic-vector-abi.c
 create mode 100644 clang/test/CodeGenCXX/x86_64-empty-base-vector-abi.cpp

diff --git a/clang/include/clang/Basic/ABIVersions.def b/clang/include/clang/Basic/ABIVersions.def
index 399be760fc7a3a..f1bf2727d9651a 100644
--- a/clang/include/clang/Basic/ABIVersions.def
+++ b/clang/include/clang/Basic/ABIVersions.def
@@ -147,6 +147,11 @@ ABI_VER_MAJOR(22)
 
 /// Attempt to be ABI-compatible with code generated by Clang 23.0.x.
 /// This causes clang to:
+///   - Use the pre-Clang 24 x86_64 SysV argument and return classifier.
+///     This preserves older, but incorrect according to the SysV spec, vector
+///     and aggregate signatures using the historical two-eightbyte model
+///     instead of classifying every eightbyte in larger vector and aggregate
+///     types.
 ///   - Ignore per-function target attributes when determining the x86 AVX ABI
 ///     level.
 ///   - On SPARC, pass a `_Complex` value with an integer element type
diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index 5324c8d366b829..64906ea4afcf92 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -8,9 +8,13 @@
 
 #include "ABIInfoImpl.h"
 #include "TargetInfo.h"
+#include "clang/AST/TypeBase.h"
 #include "clang/Basic/DiagnosticFrontend.h"
 #include "clang/Basic/SourceLocation.h"
 #include "llvm/ADT/SmallBitVector.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/ErrorHandling.h"
+#include <cstdint>
 
 using namespace clang;
 using namespace clang::CodeGen;
@@ -1333,22 +1337,6 @@ class X86_64ABIInfo : public ABIInfo {
   /// should just return Memory for the aggregate).
   static Class merge(Class Accum, Class Field);
 
-  /// postMerge - Implement the X86_64 ABI post merging algorithm.
-  ///
-  /// Post merger cleanup, reduces a malformed Hi and Lo pair to
-  /// final MEMORY or SSE classes when necessary.
-  ///
-  /// \param AggregateSize - The size of the current aggregate in
-  /// the classification process.
-  ///
-  /// \param Lo - The classification for the parts of the type
-  /// residing in the low word of the containing object.
-  ///
-  /// \param Hi - The classification for the parts of the type
-  /// residing in the higher words of the containing object.
-  ///
-  void postMerge(unsigned AggregateSize, Class &Lo, Class &Hi) const;
-
   /// classify - Determine the x86_64 register classes in which the
   /// given type T should be passed.
   ///
@@ -1375,8 +1363,17 @@ class X86_64ABIInfo : public ABIInfo {
   ///
   /// If the \arg Lo class is ComplexX87, then the \arg Hi class will
   /// also be ComplexX87.
-  void classify(QualType T, uint64_t OffsetBase, Class &Lo, Class &Hi,
-                bool isNamedArg, bool IsRegCall = false) const;
+  void classify(QualType T, uint64_t OffsetBase,
+                SmallVectorImpl<Class> &EightBytes, bool isNamedArg,
+                bool isRegCall = false) const;
+
+  void classifyClang23(QualType T, uint64_t OffsetBase,
+                       SmallVectorImpl<Class> &EightBytes, bool isNamedArg,
+                       bool isRegCall) const;
+
+  void classifyClang24(QualType T, uint64_t OffsetBase,
+                       SmallVectorImpl<Class> &EightBytes,
+                       bool isNamedArg) const;
 
   llvm::Type *GetByteVectorType(QualType Ty) const;
   llvm::Type *GetSSETypeAtOffset(llvm::Type *IRType,
@@ -1459,6 +1456,16 @@ class X86_64ABIInfo : public ABIInfo {
     return true;
   }
 
+  // For Clang 24 the classification algorithm was refactored to properly
+  // implement psABI spec, locking these platforms to the old classification
+  // algorithm to preserve ABI compatibility
+  bool useLegacyClassificationAlgorithm() const {
+    const llvm::Triple &Triple = getTarget().getTriple();
+    return getContext().getLangOpts().getClangABICompat() <=
+               LangOptions::ClangABI::Ver23 ||
+           Triple.isOSDarwin() || Triple.isPS() || Triple.isOSFreeBSD();
+  }
+
   X86AVXABILevel AVXLevel;
   // Some ABIs (e.g. X32 ABI and Native Client OS) use 32 bit pointers on
   // 64-bit hardware.
@@ -1895,39 +1902,6 @@ void WinX86_64TargetCodeGenInfo::setTargetAttributes(
   addStackProbeTargetAttributes(D, GV, CGM);
 }
 
-void X86_64ABIInfo::postMerge(unsigned AggregateSize, Class &Lo,
-                              Class &Hi) const {
-  // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
-  //
-  // (a) If one of the classes is Memory, the whole argument is passed in
-  //     memory.
-  //
-  // (b) If X87UP is not preceded by X87, the whole argument is passed in
-  //     memory.
-  //
-  // (c) If the size of the aggregate exceeds two eightbytes and the first
-  //     eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole
-  //     argument is passed in memory. NOTE: This is necessary to keep the
-  //     ABI working for processors that don't support the __m256 type.
-  //
-  // (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.
-  //
-  // Some of these are enforced by the merging logic.  Others can arise
-  // only with unions; for example:
-  //   union { _Complex double; unsigned; }
-  //
-  // Note that clauses (b) and (c) were added in 0.98.
-  //
-  if (Hi == Memory)
-    Lo = Memory;
-  if (Hi == X87Up && Lo != X87 && honorsRevision0_98())
-    Lo = Memory;
-  if (AggregateSize > 128 && (Lo != SSE || Hi != SSEUp))
-    Lo = Memory;
-  if (Hi == SSEUp && Lo != SSE)
-    Hi = SSE;
-}
-
 static X86AVXABILevel getEffectiveX86AVXABILevel(CodeGenTypes &CGT,
                                                  X86AVXABILevel GlobalAVXLevel,
                                                  const FunctionDecl *FD) {
@@ -1994,17 +1968,526 @@ X86_64ABIInfo::Class X86_64ABIInfo::merge(Class Accum, Class Field) {
   return SSE;
 }
 
-void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
-                             Class &Hi, bool isNamedArg, bool IsRegCall) const {
-  // FIXME: This code can be simplified by introducing a simple value class for
-  // Class pairs with appropriate constructor methods for the various
+void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
+                             SmallVectorImpl<Class> &EightBytes,
+                             bool isNamedArg, bool IsRegCall) const {
+  if (useLegacyClassificationAlgorithm() || IsRegCall) {
+    // RegCall is a separate ABI from the SysV psABI. Keep its classification
+    // on the pre-Clang-24 rules so this SysV classifier refactor
+    // does not silently break ABI.
+    classifyClang23(Ty, OffsetBase, EightBytes, isNamedArg, IsRegCall);
+  } else {
+    classifyClang24(Ty, OffsetBase, EightBytes, isNamedArg);
+  }
+}
+
+void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
+                                    SmallVectorImpl<Class> &EightBytes,
+                                    bool isNamedArg) const {
+  // Helpers
+  auto ClassifyAsMemory = [&]() {
+    if (EightBytes.empty())
+      EightBytes.push_back(Memory);
+    else
+      llvm::fill(EightBytes, Memory);
+  };
+
+  auto PostMerge = [&]() {
+    if (llvm::is_contained(EightBytes, Memory)) {
+      ClassifyAsMemory();
+      return;
+    }
+
+    for (unsigned I = 1, E = EightBytes.size(); I != E; ++I) {
+      if (EightBytes[I] == X87Up && EightBytes[I - 1] != X87) {
+        assert(EightBytes.size() == 2 && "Expected two eightbytes");
+        ClassifyAsMemory();
+        return;
+      }
+    }
+
+    if (EightBytes.size() > 2) {
+      if (EightBytes.front() != SSE) {
+        ClassifyAsMemory();
+        return;
+      }
+
+      for (Class C : drop_begin(EightBytes)) {
+        if (C != SSEUp) {
+          ClassifyAsMemory();
+          return;
+        }
+      }
+    }
+
+    for (unsigned I = 1, E = EightBytes.size(); I != E; ++I) {
+      if (EightBytes[I] == SSEUp && EightBytes[I - 1] != SSE &&
+          EightBytes[I - 1] != SSEUp) {
+        EightBytes[I] = SSE;
+      }
+    }
+  };
+
+  auto MergeIntoEightByte = [&](uint64_t Offset, Class C) {
+    if (C == NoClass)
+      return;
+    assert(Offset / 64 < EightBytes.size() &&
+           "classification offset is outside the containing type");
+    EightBytes[Offset / 64] = merge(EightBytes[Offset / 64], C);
+  };
+
+  auto MergeIntoAdjacentEightBytes =
+      [&](uint64_t Offset, SmallVector<Class> ToBeMergedEightBytes) {
+        uint64_t currentOffset = Offset;
+        for (Class ToBeMergedEightByte : ToBeMergedEightBytes) {
+          MergeIntoEightByte(currentOffset, ToBeMergedEightByte);
+          currentOffset = llvm::alignDown(currentOffset, uint64_t(64)) + 64;
+        }
+      };
+
+  auto SetEightByte = [&](uint64_t Offset, Class C) {
+    EightBytes[Offset / 64] = C;
+  };
+
+  auto SetAdjacentEightBytes = [&](uint64_t Offset, Class LoClass,
+                                   Class HiClass) {
+    SetEightByte(Offset, LoClass);
+    SetEightByte(llvm::alignDown(Offset, uint64_t(64)) + 64, HiClass);
+  };
+
+  auto SetTouchedEightBytes = [&](uint64_t Size, Class C) {
+    for (uint64_t Offset = OffsetBase, End = OffsetBase + Size; Offset < End;
+         Offset = llvm::alignDown(Offset, uint64_t(64)) + 64)
+      SetEightByte(Offset, C);
+  };
+
+  // End Helpers
+
+  // Resize the EigthBytes vector if needed to fit the type
+  uint64_t Size = getContext().getTypeSize(Ty);
+  unsigned Required = (OffsetBase + Size + 63) / 64;
+  if (EightBytes.size() < Required)
+    EightBytes.resize(Required, NoClass);
+
+  if (Size == 0) {
+    EightBytes.resize(1, NoClass);
+    return; // No need to classify empty types, return NoClass
+  }
+
+  // Type classification begin
+
+  // Overflow behaviour types
+  if (const OverflowBehaviorType *OBT = Ty->getAs<OverflowBehaviorType>()) {
+    classifyClang24(OBT->getUnderlyingType(), OffsetBase, EightBytes,
+                    isNamedArg);
+    return;
+  }
+
+  // Atomic types usually classify the same way as their value type. If Clang
+  // promotes the atomic storage size, fall back to memory so the full storage
+  // representation is preserved across the ABI boundary.
+  if (const AtomicType *AT = Ty->getAs<AtomicType>()) {
+    QualType ValueTy = AT->getValueType();
+    if (Size != getContext().getTypeSize(ValueTy)) {
+      ClassifyAsMemory();
+      return;
+    }
+
+    classifyClang24(ValueTy, OffsetBase, EightBytes, isNamedArg);
+    return;
+  }
+
+  // Built In Types
+  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
+    BuiltinType::Kind K = BT->getKind();
+
+    if (K == BuiltinType::Void)
+      return;
+    if (K == BuiltinType::Int128 || K == BuiltinType::UInt128) {
+      SetAdjacentEightBytes(OffsetBase, Integer, Integer);
+      return;
+    }
+    if (K >= BuiltinType::Bool && K <= BuiltinType::LongLong) {
+      SetEightByte(OffsetBase, Integer);
+      return;
+    }
+    if (K == BuiltinType::Float || K == BuiltinType::Double ||
+        K == BuiltinType::Float16 || K == BuiltinType::BFloat16) {
+      SetEightByte(OffsetBase, SSE);
+      return;
+    }
+    if (K == BuiltinType::Float128) {
+      SetAdjacentEightBytes(OffsetBase, SSE, SSEUp);
+      return;
+    }
+    if (K == BuiltinType::LongDouble) {
+      const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
+      if (LDF == &llvm::APFloat::IEEEquad())
+        SetAdjacentEightBytes(OffsetBase, SSE, SSEUp);
+      else if (LDF == &llvm::APFloat::x87DoubleExtended())
+        SetAdjacentEightBytes(OffsetBase, X87, X87Up);
+      else if (LDF == &llvm::APFloat::IEEEdouble())
+        SetEightByte(OffsetBase, SSE);
+      else
+        llvm_unreachable("unexpected long double representation!");
+      return;
+    }
+    ClassifyAsMemory();
+    return;
+  }
+
+  // Enums
+  if (const auto *ED = Ty->getAsEnumDecl()) {
+    // Classify the underlying integer type.
+    classifyClang24(ED->getIntegerType(), OffsetBase, EightBytes, isNamedArg);
+    return;
+  }
+
+  // Ptrs
+  if (Ty->hasPointerRepresentation() || Ty->isPipeType()) {
+    SetEightByte(OffsetBase, Integer);
+    return;
+  }
+
+  // C++ pointer-to-member types have integer-class storage: data member
+  // pointers are one ptrdiff_t, and member function pointers are two.
+  if (Ty->isMemberPointerType()) {
+    SetTouchedEightBytes(Size, Integer);
+    return;
+  }
+
+  // Vector & Matrix types
+  // Matrix types are "flattened" and then classified as a m*n vector with the
+  // same element type.
+  const VectorType *VT = Ty->getAs<VectorType>();
+  const MatrixType *MT = Ty->getAs<MatrixType>();
+  bool IsVectorOrMatrix = VT != nullptr || MT != nullptr;
+  if (IsVectorOrMatrix) {
+    QualType ElementType;
+    bool IsSingleElementVector = false;
+    if (VT) {
+      ElementType = VT->getElementType();
+      IsSingleElementVector = VT->getNumElements() == 1;
+    } else if (MT) {
+      ElementType = MT->getElementType();
+    } else {
+      llvm_unreachable("invalid vector or matrix type");
+    }
+    ElementType = getContext().getCanonicalType(ElementType);
+
+    auto IsFPVectorElement = [&]() {
+      return ElementType->isFloat16Type() || ElementType->isBFloat16Type() ||
+             ElementType == getContext().FloatTy ||
+             ElementType == getContext().DoubleTy;
+    };
+
+    auto SetEightByteClasses = [&](Class FirstClass, Class RestClass) {
+      // Vector objects should not start partway through one eightbyte and
+      // cross into another here. Aggregate classification rejects unaligned
+      // non-bit-field vector fields as MEMORY before recursing. Matrix types
+      // use element alignment, so they can still have a partial first chunk.
+      [[maybe_unused]] bool HasPartialFirstEightByte =
+          OffsetBase % 64 && OffsetBase / 64 != (OffsetBase + Size - 1) / 64;
+      if (VT)
+        assert(!HasPartialFirstEightByte &&
+               "unaligned vector should have been classified as memory");
+
+      SetEightByte(OffsetBase, FirstClass);
+      for (uint64_t Offset = llvm::alignDown(OffsetBase, uint64_t(64)) + 64,
+                    End = OffsetBase + Size;
+           Offset < End; Offset += 64)
+        SetEightByte(Offset, RestClass);
+    };
+
+    if (IsSingleElementVector && IsFPVectorElement()) {
+      // GCC passes single-element floating-point vectors in memory. This is a
+      // compatibility rule, not a psABI classification rule.
+      ClassifyAsMemory();
+      return;
+    }
+
+    // Vectors of __int128 wider than 128 bits classify as MEMORY: each
+    // __int128 lane contributes two INTEGER eightbytes, and post-merge cleanup
+    // only keeps wider objects in registers when they are SSE followed by
+    // SSEUp.
+    if (passInt128VectorsInMem() && Size > 128 &&
+        (ElementType->isSpecificBuiltinType(BuiltinType::Int128) ||
+         ElementType->isSpecificBuiltinType(BuiltinType::UInt128))) {
+      ClassifyAsMemory();
+      return;
+    }
+
+    // The psABI explicitly classifies named vector types like __m64 and __m128
+    // as SSE/SSEUP, but it does not spell out arbitrary GNU vector extension
+    // integer vectors. We decided to match GCC 15.2.0 which classifies
+    // integer vectors up to 32 bits as INTEGER, while 64-bit and wider integer
+    // vectors use SSE/SSEUP vector classes.
+    if (Size <= 32) {
+      // The ABI classifies floating-point vector elements as SSE. Small
+      // integer vectors are passed as INTEGER, matching GCC for:
+      // 4 bytes - <4 x char>, <2 x short>, <1 x int>
+      // 2 bytes - <2 x char>, <1 x short>
+      // 1 byte  - <1 x char>
+      Class C = IsFPVectorElement() ? SSE : Integer;
+
+      SetEightByteClasses(C, C);
+    } else if (Size == 64) {
+      SetEightByteClasses(SSE, SSE);
+    } else if (Size == 128 ||
+               (isNamedArg && Size <= getNativeVectorSizeForAVXABI(AVXLevel))) {
+      SetEightByteClasses(SSE, SSEUp);
+    } else {
+      // Any other types that are not explicity handled should be passed as
+      // memory
+      ClassifyAsMemory();
+    }
+
+    return;
+  }
+
+  // Complex types
+  if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
+    QualType ET = getContext().getCanonicalType(CT->getElementType());
+
+    uint64_t Size = getContext().getTypeSize(Ty);
+    if (ET->isIntegralOrEnumerationType()) {
+      if (Size <= 64)
+        SetEightByte(OffsetBase, Integer);
+      else if (Size <= 128)
+        SetAdjacentEightBytes(OffsetBase, Integer, Integer);
+    } else if (ET->isFloat16Type() || ET == getContext().FloatTy ||
+               ET->isBFloat16Type()) {
+      SetEightByte(OffsetBase, SSE);
+    } else if (ET == getContext().DoubleTy) {
+      SetAdjacentEightBytes(OffsetBase, SSE, SSE);
+    } else if (ET == getContext().LongDoubleTy) {
+      const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
+      if (LDF == &llvm::APFloat::IEEEquad())
+        ClassifyAsMemory();
+      else if (LDF == &llvm::APFloat::x87DoubleExtended())
+        SetEightByte(OffsetBase, ComplexX87);
+      else if (LDF == &llvm::APFloat::IEEEdouble())
+        SetAdjacentEightBytes(OffsetBase, SSE, SSE);
+      else
+        llvm_unreachable("unexpected long double representation!");
+    } else if (ET->isFloat128Type()) {
+      ClassifyAsMemory();
+    }
+
+    // If this complex type crosses an eightbyte boundary then it
+    // should be split.
+    uint64_t EB_Real = (OffsetBase) / 64;
+    uint64_t EB_Imag = (OffsetBase + getContext().getTypeSize(ET)) / 64;
+    if (EightBytes[EB_Imag] == NoClass && EB_Real != EB_Imag)
+      SetAdjacentEightBytes(OffsetBase, EightBytes[EB_Real],
+                            EightBytes[EB_Real]);
+
+    return;
+  }
+
+  // BitInt Types
+  if (const auto *EITy = Ty->getAs<BitIntType>()) {
+    if (EITy->getNumBits() <= 64)
+      SetEightByte(OffsetBase, Integer);
+    else if (EITy->getNumBits() <= 128)
+      SetAdjacentEightBytes(OffsetBase, Integer, Integer);
+    else {
+      // Larger values need to get passed in memory.
+      ClassifyAsMemory();
+    }
+
+    return;
+  }
+
+  // Arrays
+  if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
+    // Arrays are treated like structures.
+
+    // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
+    // than eight eightbytes, ..., it has class MEMORY.
+    if (Size > 512) {
+      ClassifyAsMemory();
+      return;
+    }
+
+    // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
+    // fields, it has class MEMORY.
+    //
+    // Only need to check alignment of array base.
+    if (OffsetBase % getContext().getTypeAlign(AT->getElementType())) {
+      ClassifyAsMemory();
+      return;
+    }
+
+    // Otherwise implement simplified merge. We could be smarter about
+    // this, but it isn't worth it and would be harder to verify.
+    uint64_t EltSize = getContext().getTypeSize(AT->getElementType());
+    uint64_t ArraySize = AT->getZExtSize();
+
+    for (uint64_t i = 0, Offset = OffsetBase; i < ArraySize;
+         ++i, Offset += EltSize) {
+      SmallVector<Class> ArrayElEightBytes;
+      classifyClang24(AT->getElementType(), Offset % 64, ArrayElEightBytes,
+                      isNamedArg);
+      MergeIntoAdjacentEightBytes(Offset, ArrayElEightBytes);
+      if (llvm::is_contained(EightBytes, Memory))
+        break;
+    }
+
+    PostMerge();
+    return;
+  }
+
+  // CXX Record Type
+  if (const RecordType *RT = Ty->getAsCanonical<RecordType>()) {
+    // uint64_t Size = getContext().getTypeSize(Ty);
+
+    // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
+    // than eight eightbytes, ..., it has class MEMORY.
+    if (Size > 512) {
+      ClassifyAsMemory();
+      return;
+    }
+
+    // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial
+    // copy constructor or a non-trivial destructor, it is passed by
+    // invisible reference.
+    if (getRecordArgABI(RT, getCXXABI())) {
+      ClassifyAsMemory();
+      return;
+    }
+
+    const RecordDecl *RD = RT->getDecl()->getDefinitionOrSelf();
+
+    // Assume variable sized types are passed in memory.
+    if (RD->hasFlexibleArrayMember()) {
+      ClassifyAsMemory();
+      return;
+    }
+
+    const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
+
+    // If this is a C++ record, classify the bases first.
+    if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
+      for (const auto &I : CXXRD->bases()) {
+        assert(!I.isVirtual() && !I.getType()->isDependentType() &&
+               "Unexpected base class!");
+        const auto *Base = I.getType()->castAsCXXRecordDecl();
+
+        // Classify this field.
+        //
+        // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds
+        // a single eightbyte, each is classified separately. Each
+        // eightbyte gets initialized to class NO_CLASS.
+        SmallVector<Class> BaseEightBytes;
+        uint64_t Offset =
+            OffsetBase + getContext().toBits(Layout.getBaseClassOffset(Base));
+        classifyClang24(I.getType(), Offset % 64, BaseEightBytes, isNamedArg);
+        // If the base is memory the whole record will be passed as memory
+        if (BaseEightBytes[0] == Memory) {
+          ClassifyAsMemory();
+          return;
+        }
+        MergeIntoAdjacentEightBytes(Offset, BaseEightBytes);
+      }
+    }
+
+    // Classify the fields one at a time, merging the results.
+    unsigned idx = 0;
+
+    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
+         i != e; ++i, ++idx) {
+      uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
+      bool BitField = i->isBitField();
+
+      // Ignore padding bit-fields.
+      if (BitField && i->isUnnamedBitField())
+        continue;
+
+      bool IsInMemory =
+          Offset % getContext().getTypeAlign(i->getType().getCanonicalType());
+      // Note, skip this test for bit-fields, see below.
+      if (!BitField && IsInMemory) {
+        ClassifyAsMemory();
+        PostMerge();
+        return;
+      }
+
+      // Classify this field.
+      //
+      // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate
+      // exceeds a single eightbyte, each is classified
+      // separately. Each eightbyte gets initialized to class
+      // NO_CLASS.
+      // Bit-fields require special handling, they do not force the
+      // structure to be passed in memory even if unaligned, and
+      // therefore they can straddle an eightbyte.
+      if (BitField) {
+        assert(!i->isUnnamedBitField());
+        uint64_t BitSize = i->getBitWidthValue();
+        for (uint64_t BitOffset = Offset, End = Offset + BitSize;
+             BitOffset < End;
+             BitOffset = llvm::alignDown(BitOffset, uint64_t(64)) + 64)
+          MergeIntoEightByte(BitOffset, Integer);
+      } else {
+        SmallVector<Class> FieldEightBytes;
+        classifyClang24(i->getType(), Offset % 64, FieldEightBytes, isNamedArg);
+        MergeIntoAdjacentEightBytes(Offset, FieldEightBytes);
+      }
+
+      if (llvm::is_contained(EightBytes, Memory))
+        break;
+    }
+
+    PostMerge();
+  }
+}
+
+void X86_64ABIInfo::classifyClang23(QualType Ty, uint64_t OffsetBase,
+                                    SmallVectorImpl<Class> &EightBytes,
+                                    bool isNamedArg, bool IsRegCall) const {
+  // FIXME: This code can be simplified by introducing a simple value class
+  // for Class pairs with appropriate constructor methods for the various
   // situations.
 
   // FIXME: Some of the split computations are wrong; unaligned vectors
   // shouldn't be passed in registers for example, so there is no chance they
   // can straddle an eightbyte. Verify & simplify.
+  if (EightBytes.empty())
+    EightBytes.resize(2, NoClass);
 
-  Lo = Hi = NoClass;
+  Class &Lo = EightBytes[0];
+  Class &Hi = EightBytes[1];
+  auto PostMerge = [&](unsigned AggregateSize) {
+    // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
+    //
+    // (a) If one of the classes is Memory, the whole argument is passed in
+    //     memory.
+    //
+    // (b) If X87UP is not preceded by X87, the whole argument is passed in
+    //     memory.
+    //
+    // (c) If the size of the aggregate exceeds two eightbytes and the first
+    //     eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole
+    //     argument is passed in memory. NOTE: This is necessary to keep the
+    //     ABI working for processors that don't support the __m256 type.
+    //
+    // (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.
+    //
+    // Some of these are enforced by the merging logic. Others can arise only
+    // with unions; for example:
+    //   union { _Complex double; unsigned; }
+    //
+    // Note that clauses (b) and (c) were added in 0.98.
+    if (Hi == Memory)
+      Lo = Memory;
+    if (Hi == X87Up && Lo != X87 && honorsRevision0_98())
+      Lo = Memory;
+    if (AggregateSize > 128 && (Lo != SSE || Hi != SSEUp))
+      Lo = Memory;
+    if (Hi == SSEUp && Lo != SSE)
+      Hi = SSE;
+  };
 
   Class &Current = OffsetBase < 64 ? Lo : Hi;
   Current = Memory;
@@ -2045,7 +2528,9 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
 
   if (const auto *ED = Ty->getAsEnumDecl()) {
     // Classify the underlying integer type.
-    classify(ED->getIntegerType(), OffsetBase, Lo, Hi, isNamedArg);
+    SmallVector<Class> IntEightBytes = {Lo, Hi};
+    classifyClang23(ED->getIntegerType(), OffsetBase, IntEightBytes, isNamedArg,
+                    IsRegCall);
     return;
   }
 
@@ -2126,9 +2611,9 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
         return;
 
       // Arguments of 256-bits are split into four eightbyte chunks. The
-      // least significant one belongs to class SSE and all the others to class
-      // SSEUP. The original Lo and Hi design considers that types can't be
-      // greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.
+      // least significant one belongs to class SSE and all the others to
+      // class SSEUP. The original Lo and Hi design considers that types can't
+      // be greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.
       // This design isn't correct for 256-bits, but since there're no cases
       // where the upper parts would need to be inspected, avoid adding
       // complexity and just consider Hi to match the 64-256 part.
@@ -2137,8 +2622,8 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
       // registers if they are "named", i.e. not part of the "..." of a
       // variadic function.
       //
-      // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args are
-      // split into eight eightbyte chunks, one SSE and seven SSEUP.
+      // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args
+      // are split into eight eightbyte chunks, one SSE and seven SSEUP.
       Lo = SSE;
       Hi = SSEUp;
     }
@@ -2219,20 +2704,25 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
     // contains a single 256-bit element. Since Lo and Hi logic isn't extended
     // to work for sizes wider than 128, early check and fallback to memory.
     //
+    // Note: There is a bug here, the assumption is that this is returning
+    // "Memory" but actually it is returning "NoClass" because Current is set to
+    // NoClass above. Fixed in the Clang 24 implementation.
     if (Size > 128 &&
         (Size != EltSize || Size > getNativeVectorSizeForAVXABI(AVXLevel)))
       return;
 
-    for (uint64_t i=0, Offset=OffsetBase; i<ArraySize; ++i, Offset += EltSize) {
-      Class FieldLo, FieldHi;
-      classify(AT->getElementType(), Offset, FieldLo, FieldHi, isNamedArg);
-      Lo = merge(Lo, FieldLo);
-      Hi = merge(Hi, FieldHi);
+    for (uint64_t i = 0, Offset = OffsetBase; i < ArraySize;
+         ++i, Offset += EltSize) {
+      SmallVector<Class> FieldEightBytes = {NoClass, NoClass};
+      classifyClang23(AT->getElementType(), Offset, FieldEightBytes, isNamedArg,
+                      IsRegCall);
+      Lo = merge(Lo, FieldEightBytes[0]);
+      Hi = merge(Hi, FieldEightBytes[1]);
       if (Lo == Memory || Hi == Memory)
         break;
     }
 
-    postMerge(Size, Lo, Hi);
+    PostMerge(Size);
     assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp array classification.");
     return;
   }
@@ -2268,27 +2758,33 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
         assert(!I.isVirtual() && !I.getType()->isDependentType() &&
                "Unexpected base class!");
         const auto *Base = I.getType()->castAsCXXRecordDecl();
+
         // Classify this field.
         //
         // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a
-        // single eightbyte, each is classified separately. Each eightbyte gets
-        // initialized to class NO_CLASS.
-        Class FieldLo, FieldHi;
+        // single eightbyte, each is classified separately. Each eightbyte
+        // gets initialized to class NO_CLASS.
+        SmallVector<Class> FieldEightBytes = {NoClass, NoClass};
         uint64_t Offset =
-          OffsetBase + getContext().toBits(Layout.getBaseClassOffset(Base));
-        classify(I.getType(), Offset, FieldLo, FieldHi, isNamedArg);
-        Lo = merge(Lo, FieldLo);
-        Hi = merge(Hi, FieldHi);
+            OffsetBase + getContext().toBits(Layout.getBaseClassOffset(Base));
+        classifyClang23(I.getType(), Offset, FieldEightBytes, isNamedArg,
+                        IsRegCall);
+        // Note: There is a bug here for types larger than 128bits. Doesn't
+        // properly implement the SYSV classification algorithm. Fixed in
+        // Clang 24 implementation.
+        Lo = merge(Lo, FieldEightBytes[0]);
+        Hi = merge(Hi, FieldEightBytes[1]);
         if (returnCXXRecordGreaterThan128InMem() &&
             !isEmptyRecord(getContext(), I.getType(), true) &&
             (Size > 128 && (Size != getContext().getTypeSize(I.getType()) ||
                             Size > getNativeVectorSizeForAVXABI(AVXLevel)))) {
-          // The only case a 256(or 512)-bit wide vector could be used to return
-          // is when CXX record contains a single 256(or 512)-bit element.
+          // The only case a 256(or 512)-bit wide vector could be used to
+          // return is when CXX record contains a single 256(or 512)-bit
+          // element.
           Lo = Memory;
         }
         if (Lo == Memory || Hi == Memory) {
-          postMerge(Size, Lo, Hi);
+          PostMerge(Size);
           return;
         }
       }
@@ -2306,7 +2802,7 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
     bool IsUnion = RT->isUnionType() && !UseClang11Compat;
 
     for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
-           i != e; ++i, ++idx) {
+         i != e; ++i, ++idx) {
       uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
       bool BitField = i->isBitField();
 
@@ -2320,11 +2816,12 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
         continue;
 
       // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger than
-      // eight eightbytes, or it contains unaligned fields, it has class MEMORY.
+      // eight eightbytes, or it contains unaligned fields, it has class
+      // MEMORY.
       //
-      // The only case a 256-bit or a 512-bit wide vector could be used is when
-      // the struct contains a single 256-bit or 512-bit element. Early check
-      // and fallback to memory.
+      // The only case a 256-bit or a 512-bit wide vector could be used is
+      // when the struct contains a single 256-bit or 512-bit element. Early
+      // check and fallback to memory.
       //
       // FIXME: Extended the Lo and Hi logic properly to work for size wider
       // than 128.
@@ -2332,7 +2829,7 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
           ((!IsUnion && Size != getContext().getTypeSize(i->getType())) ||
            Size > getNativeVectorSizeForAVXABI(AVXLevel))) {
         Lo = Memory;
-        postMerge(Size, Lo, Hi);
+        PostMerge(Size);
         return;
       }
 
@@ -2341,7 +2838,7 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
       // Note, skip this test for bit-fields, see below.
       if (!BitField && IsInMemory) {
         Lo = Memory;
-        postMerge(Size, Lo, Hi);
+        PostMerge(Size);
         return;
       }
 
@@ -2351,7 +2848,7 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
       // exceeds a single eightbyte, each is classified
       // separately. Each eightbyte gets initialized to class
       // NO_CLASS.
-      Class FieldLo, FieldHi;
+      SmallVector<Class> FieldEightBytes = {NoClass, NoClass};
 
       // Bit-fields require special handling, they do not force the
       // structure to be passed in memory even if unaligned, and
@@ -2367,28 +2864,33 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
 
         if (EB_Lo) {
           assert(EB_Hi == EB_Lo && "Invalid classification, type > 16 bytes.");
-          FieldLo = NoClass;
-          FieldHi = Integer;
+          FieldEightBytes[0] = NoClass;
+          FieldEightBytes[1] = Integer;
         } else {
-          FieldLo = Integer;
-          FieldHi = EB_Hi ? Integer : NoClass;
+          FieldEightBytes[0] = Integer;
+          FieldEightBytes[1] = EB_Hi ? Integer : NoClass;
         }
       } else
-        classify(i->getType(), Offset, FieldLo, FieldHi, isNamedArg);
-      Lo = merge(Lo, FieldLo);
-      Hi = merge(Hi, FieldHi);
+        classifyClang23(i->getType(), Offset, FieldEightBytes, isNamedArg,
+                        IsRegCall);
+      Lo = merge(Lo, FieldEightBytes[0]);
+      Hi = merge(Hi, FieldEightBytes[1]);
       if (Lo == Memory || Hi == Memory)
         break;
     }
 
-    postMerge(Size, Lo, Hi);
+    PostMerge(Size);
   }
 }
 
 ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {
   // If this is a scalar LLVM value then assume LLVM will pass it in the right
   // place naturally.
-  if (!isAggregateTypeForABI(Ty)) {
+  // The legacy classifier relied on this fallback for ABI compatibility, even
+  // for vector types which it had classified as memory. The Clang 24
+  // classifier instead expects a memory classification to remain indirect.
+  if (!isAggregateTypeForABI(Ty) &&
+      (useLegacyClassificationAlgorithm() || !IsIllegalVectorType(Ty))) {
     // Treat an enum type as its underlying type.
     if (const auto *ED = Ty->getAsEnumDecl())
       Ty = ED->getIntegerType();
@@ -2405,6 +2907,13 @@ ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {
 }
 
 bool X86_64ABIInfo::IsIllegalVectorType(QualType Ty) const {
+  if (const AtomicType *AT = Ty->getAs<AtomicType>()) {
+    // Preserve the pre-Clang 24 direct lowering of atomic vector types.
+    if (useLegacyClassificationAlgorithm())
+      return false;
+    Ty = AT->getValueType();
+  }
+
   if (const VectorType *VecTy = Ty->getAs<VectorType>()) {
     uint64_t Size = getContext().getTypeSize(VecTy);
     unsigned LargestVector = getNativeVectorSizeForAVXABI(AVXLevel);
@@ -2427,9 +2936,9 @@ ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty,
   //
   // This assumption is optimistic, as there could be free registers available
   // when we need to pass this argument in memory, and LLVM could try to pass
-  // the argument in the free register. This does not seem to happen currently,
-  // but this code would be much safer if we could mark the argument with
-  // 'onstack'. See PR12193.
+  // the argument in the free register. This does not seem to happen
+  // currently, but this code would be much safer if we could mark the
+  // argument with 'onstack'. See PR12193.
   if (!isAggregateTypeForABI(Ty) && !IsIllegalVectorType(Ty) &&
       !Ty->isBitIntType()) {
     // Treat an enum type as its underlying type.
@@ -2455,18 +2964,18 @@ ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty,
   // We do this by coercing the value into a scalar type which the backend can
   // handle naturally (i.e., without using byval).
   //
-  // For simplicity, we currently only do this when we have exhausted all of the
-  // free integer registers. Doing this when there are free integer registers
-  // would require more care, as we would have to ensure that the coerced value
-  // did not claim the unused register. That would require either reording the
-  // arguments to the function (so that any subsequent inreg values came first),
-  // or only doing this optimization when there were no following arguments that
-  // might be inreg.
+  // For simplicity, we currently only do this when we have exhausted all of
+  // the free integer registers. Doing this when there are free integer
+  // registers would require more care, as we would have to ensure that the
+  // coerced value did not claim the unused register. That would require
+  // either reording the arguments to the function (so that any subsequent
+  // inreg values came first), or only doing this optimization when there were
+  // no following arguments that might be inreg.
   //
   // We currently expect it to be rare (particularly in well written code) for
   // arguments to be passed on the stack when there are still free integer
-  // registers available (this would typically imply large structs being passed
-  // by value), so this seems like a fair tradeoff for now.
+  // registers available (this would typically imply large structs being
+  // passed by value), so this seems like a fair tradeoff for now.
   //
   // We can revisit this if the backend grows support for 'onstack' parameter
   // attributes. See PR12193.
@@ -2476,8 +2985,8 @@ ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty,
     // If this type fits in an eightbyte, coerce it into the matching integral
     // type, which will end up on the stack (with alignment 8).
     if (Align == 8 && Size <= 64)
-      return ABIArgInfo::getDirect(llvm::IntegerType::get(getVMContext(),
-                                                          Size));
+      return ABIArgInfo::getDirect(
+          llvm::IntegerType::get(getVMContext(), Size));
   }
 
   return ABIArgInfo::getIndirect(CharUnits::fromQuantity(Align),
@@ -2514,7 +3023,6 @@ llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const {
   uint64_t Size = getContext().getTypeSize(Ty);
   assert((Size == 128 || Size == 256 || Size == 512) && "Invalid type found!");
 
-
   // Return a LLVM IR vector type based on the size of 'Ty'.
   return llvm::FixedVectorType::get(llvm::Type::getDoubleTy(getVMContext()),
                                     Size / 64);
@@ -2522,9 +3030,10 @@ llvm::Type *X86_64ABIInfo::GetByteVectorType(QualType Ty) const {
 
 /// BitsContainNoUserData - Return true if the specified [start,end) bit range
 /// is known to either be off the end of the specified type or being in
-/// alignment padding.  The user type specified is known to be at most 128 bits
-/// in size, and have passed through X86_64ABIInfo::classify with a successful
-/// classification that put one of the two halves in the INTEGER class.
+/// alignment padding.  The user type specified is known to be at most 128
+/// bits in size, and have passed through X86_64ABIInfo::classify with a
+/// successful classification that put one of the two halves in the INTEGER
+/// class.
 ///
 /// It is conservatively correct to return false.
 static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
@@ -2540,15 +3049,17 @@ static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
     unsigned EltSize = (unsigned)Context.getTypeSize(AT->getElementType());
     unsigned NumElts = (unsigned)AT->getZExtSize();
 
-    // Check each element to see if the element overlaps with the queried range.
+    // Check each element to see if the element overlaps with the queried
+    // range.
     for (unsigned i = 0; i != NumElts; ++i) {
       // If the element is after the span we care about, then we're done..
-      unsigned EltOffset = i*EltSize;
-      if (EltOffset >= EndBit) break;
+      unsigned EltOffset = i * EltSize;
+      if (EltOffset >= EndBit)
+        break;
 
-      unsigned EltStart = EltOffset < StartBit ? StartBit-EltOffset :0;
+      unsigned EltStart = EltOffset < StartBit ? StartBit - EltOffset : 0;
       if (!BitsContainNoUserData(AT->getElementType(), EltStart,
-                                 EndBit-EltOffset, Context))
+                                 EndBit - EltOffset, Context))
         return false;
     }
     // If it overlaps no elements, then it is safe to process as padding.
@@ -2567,16 +3078,17 @@ static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
 
         // If the base is after the span we care about, ignore it.
         unsigned BaseOffset = Context.toBits(Layout.getBaseClassOffset(Base));
-        if (BaseOffset >= EndBit) continue;
+        if (BaseOffset >= EndBit)
+          continue;
 
-        unsigned BaseStart = BaseOffset < StartBit ? StartBit-BaseOffset :0;
-        if (!BitsContainNoUserData(I.getType(), BaseStart,
-                                   EndBit-BaseOffset, Context))
+        unsigned BaseStart = BaseOffset < StartBit ? StartBit - BaseOffset : 0;
+        if (!BitsContainNoUserData(I.getType(), BaseStart, EndBit - BaseOffset,
+                                   Context))
           return false;
       }
     }
 
-    // Verify that no field has data that overlaps the region of interest.  Yes
+    // Verify that no field has data that overlaps the region of interest. Yes
     // this could be sped up a lot by being smarter about queried fields,
     // however we're only looking at structs up to 16 bytes, so we don't care
     // much.
@@ -2586,10 +3098,11 @@ static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
       unsigned FieldOffset = (unsigned)Layout.getFieldOffset(idx);
 
       // If we found a field after the region we care about, then we're done.
-      if (FieldOffset >= EndBit) break;
+      if (FieldOffset >= EndBit)
+        break;
 
-      unsigned FieldStart = FieldOffset < StartBit ? StartBit-FieldOffset :0;
-      if (!BitsContainNoUserData(i->getType(), FieldStart, EndBit-FieldOffset,
+      unsigned FieldStart = FieldOffset < StartBit ? StartBit - FieldOffset : 0;
+      if (!BitsContainNoUserData(i->getType(), FieldStart, EndBit - FieldOffset,
                                  Context))
         return false;
     }
@@ -2632,14 +3145,27 @@ static llvm::Type *getFPTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
   return nullptr;
 }
 
-/// GetSSETypeAtOffset - Return a type that will be passed by the backend in the
-/// low 8 bytes of an XMM register, corresponding to the SSE class.
-llvm::Type *X86_64ABIInfo::
-GetSSETypeAtOffset(llvm::Type *IRType, unsigned IROffset,
-                   QualType SourceTy, unsigned SourceOffset) const {
+/// GetSSETypeAtOffset - Return a type that will be passed by the backend in
+/// the low 8 bytes of an XMM register, corresponding to the SSE class.
+llvm::Type *X86_64ABIInfo::GetSSETypeAtOffset(llvm::Type *IRType,
+                                              unsigned IROffset,
+                                              QualType SourceTy,
+                                              unsigned SourceOffset) const {
   const llvm::DataLayout &TD = getDataLayout();
   unsigned SourceSize =
       (unsigned)getContext().getTypeSize(SourceTy) / 8 - SourceOffset;
+
+  // Keep 16-bit FP vector lanes visible when they fit in one SSE eightbyte.
+  // Reinterpreting them as double would make attributes like nofpclass describe
+  // carrier bits rather than the source vector elements.
+  if (IROffset == 0 && SourceOffset == 0) {
+    if (auto *VTy = dyn_cast<llvm::FixedVectorType>(IRType)) {
+      llvm::Type *EltTy = VTy->getElementType();
+      if (EltTy->is16bitFPTy() && TD.getTypeAllocSize(VTy) <= 8)
+        return VTy;
+    }
+  }
+
   llvm::Type *T0 = getFPTypeAtOffset(IRType, IROffset, TD);
   if (!T0 || T0->isDoubleTy())
     return llvm::Type::getDoubleTy(getVMContext());
@@ -2648,7 +3174,7 @@ GetSSETypeAtOffset(llvm::Type *IRType, unsigned IROffset,
   llvm::Type *T1 = nullptr;
   unsigned T0Size = TD.getTypeAllocSize(T0);
   if (SourceSize > T0Size)
-      T1 = getFPTypeAtOffset(IRType, IROffset + T0Size, TD);
+    T1 = getFPTypeAtOffset(IRType, IROffset + T0Size, TD);
   if (T1 == nullptr) {
     // Check if IRType is a half/bfloat + float. float type will be in IROffset+4 due
     // to its alignment.
@@ -2809,8 +3335,20 @@ GetX86_64ByValArgumentPair(llvm::Type *Lo, llvm::Type *Hi,
 ABIArgInfo X86_64ABIInfo::classifyReturnType(QualType RetTy) const {
   // AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the
   // classification algorithm.
-  X86_64ABIInfo::Class Lo, Hi;
-  classify(RetTy, 0, Lo, Hi, /*isNamedArg*/ true);
+
+  SmallVector<X86_64ABIInfo::Class> EightBytes;
+  classify(RetTy, 0, EightBytes, /*isNamedArg*/ true);
+
+  X86_64ABIInfo::Class Lo = NoClass;
+  X86_64ABIInfo::Class Hi = NoClass;
+
+  if (EightBytes.size() > 0) {
+    Lo = EightBytes[0];
+  }
+
+  if (EightBytes.size() > 1) {
+    Hi = EightBytes[1];
+  }
 
   // Check some invariants.
   assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
@@ -2944,8 +3482,19 @@ X86_64ABIInfo::classifyArgumentType(QualType Ty, unsigned freeIntRegs,
                                     bool isNamedArg, bool IsRegCall) const {
   Ty = useFirstFieldIfTransparentUnion(Ty);
 
-  X86_64ABIInfo::Class Lo, Hi;
-  classify(Ty, 0, Lo, Hi, isNamedArg, IsRegCall);
+  SmallVector<X86_64ABIInfo::Class> EightBytes;
+  classify(Ty, 0, EightBytes, isNamedArg, IsRegCall);
+
+  X86_64ABIInfo::Class Lo = NoClass;
+  X86_64ABIInfo::Class Hi = NoClass;
+
+  if (EightBytes.size() > 0) {
+    Lo = EightBytes[0];
+  }
+
+  if (EightBytes.size() > 1) {
+    Hi = EightBytes[1];
+  }
 
   // Check some invariants.
   // FIXME: Enforce these by construction.
diff --git a/clang/test/CodeGen/X86/x86-illegal-vector.c b/clang/test/CodeGen/X86/x86-illegal-vector.c
new file mode 100644
index 00000000000000..abb863f28d9451
--- /dev/null
+++ b/clang/test/CodeGen/X86/x86-illegal-vector.c
@@ -0,0 +1,22 @@
+// RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +sse2 -emit-llvm -o - | FileCheck %s --check-prefixes=REGRET128,MEMRET256,MEMRET512
+// RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +sse2 -emit-llvm -o - -fclang-abi-compat=22 | FileCheck %s --check-prefixes=REGRET128,REGRET256,REGRET512
+
+// RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +avx -emit-llvm -o - | FileCheck %s --check-prefixes=REGRET128,REGRET256,MEMRET512
+// RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +avx -emit-llvm -o - -fclang-abi-compat=22 | FileCheck %s --check-prefixes=REGRET128,REGRET256,REGRET512
+
+// RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +avx512f -emit-llvm -o - | FileCheck %s --check-prefixes=REGRET128,REGRET256,REGRET512
+// RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +avx512f -emit-llvm -o - -fclang-abi-compat=22 | FileCheck %s --check-prefixes=REGRET128,REGRET256,REGRET512
+
+#define __MM_MALLOC_H
+#include <x86intrin.h>
+
+// REGRET128: define{{.*}} <4 x float> @get2()
+__m128 get2() { return (__m128){5, 6}; }
+
+// MEMRET256: define{{.*}} void @get4(ptr{{.*}} sret(<8 x float>) align 32 %{{.*}})
+// REGRET256: define{{.*}} <8 x float> @get4()
+__m256 get4() { return (__m256){7, 8, 9, 10}; }
+
+// MEMRET512: define{{.*}} void @get8(ptr{{.*}} sret(<16 x float>) align 64 %{{.*}})
+// REGRET512: define{{.*}} <16 x float> @get8()
+__m512 get8() { return (__m512){7, 8, 9, 10, 1, 2, 3, 4}; }
diff --git a/clang/test/CodeGen/X86/x86-small-fp-vector-abi.c b/clang/test/CodeGen/X86/x86-small-fp-vector-abi.c
new file mode 100644
index 00000000000000..66f0fec6ee247f
--- /dev/null
+++ b/clang/test/CodeGen/X86/x86-small-fp-vector-abi.c
@@ -0,0 +1,58 @@
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -fclang-abi-compat=22 -emit-llvm -o - %s | FileCheck %s --check-prefix=LEGACY
+
+typedef _Float16 v1hf __attribute__((vector_size(2)));
+typedef _Float16 v2hf __attribute__((vector_size(4)));
+typedef _Float16 v4hf __attribute__((vector_size(8)));
+typedef float v1sf __attribute__((vector_size(4)));
+typedef float v2sf __attribute__((vector_size(8)));
+typedef double v1df __attribute__((vector_size(8)));
+typedef int v1si __attribute__((vector_size(4)));
+
+// Single-element floating-point vectors are passed in memory for GCC
+// compatibility.
+// CHECK-LABEL: define{{.*}} half @take_v1hf(ptr noundef byval(<1 x half>) align 8
+// LEGACY-LABEL: define{{.*}} half @take_v1hf(i16 noundef
+_Float16 take_v1hf(v1hf x) { return x[0]; }
+
+// CHECK-LABEL: define{{.*}} float @take_v1sf(ptr noundef byval(<1 x float>) align 8
+// LEGACY-LABEL: define{{.*}} float @take_v1sf(i32 noundef
+float take_v1sf(v1sf x) { return x[0]; }
+
+// CHECK-LABEL: define{{.*}} double @take_v1df(ptr noundef byval(<1 x double>) align 8
+// LEGACY-LABEL: define{{.*}} double @take_v1df(ptr noundef byval(<1 x double>) align 8
+double take_v1df(v1df x) { return x[0]; }
+
+// Single-element floating-point vector returns use memory too.
+// CHECK-LABEL: define{{.*}} void @return_v1hf(
+// CHECK-SAME: ptr {{.*}}sret(<1 x half>) align 2 %agg.result,
+// CHECK-SAME: ptr noundef byval(<1 x half>) align 8
+// LEGACY-LABEL: define{{.*}} i16 @return_v1hf(i16 noundef
+v1hf return_v1hf(v1hf x) { return x; }
+
+// CHECK-LABEL: define{{.*}} void @return_v1sf(
+// CHECK-SAME: ptr {{.*}}sret(<1 x float>) align 4 %agg.result,
+// CHECK-SAME: ptr noundef byval(<1 x float>) align 8
+// LEGACY-LABEL: define{{.*}} i32 @return_v1sf(i32 noundef
+v1sf return_v1sf(v1sf x) { return x; }
+
+// CHECK-LABEL: define{{.*}} void @return_v1df(
+// CHECK-SAME: ptr {{.*}}sret(<1 x double>) align 8 %agg.result,
+// CHECK-SAME: ptr noundef byval(<1 x double>) align 8
+// LEGACY-LABEL: define{{.*}} <1 x double> @return_v1df(
+// LEGACY-SAME: ptr noundef byval(<1 x double>) align 8
+v1df return_v1df(v1df x) { return x; }
+
+// Multi-element floating-point vectors are classified as SSE.
+// CHECK-LABEL: define{{.*}} half @take_v2hf(<2 x half> noundef
+_Float16 take_v2hf(v2hf x) { return x[0]; }
+
+// CHECK-LABEL: define{{.*}} half @take_v4hf(<4 x half> noundef
+_Float16 take_v4hf(v4hf x) { return x[0]; }
+
+// CHECK-LABEL: define{{.*}} float @take_v2sf(double noundef
+float take_v2sf(v2sf x) { return x[0]; }
+
+// Small integer vectors stay INTEGER.
+// CHECK-LABEL: define{{.*}} i32 @take_v1si(i32 noundef
+int take_v1si(v1si x) { return x[0]; }
diff --git a/clang/test/CodeGen/X86/x86-vec-i128.c b/clang/test/CodeGen/X86/x86-vec-i128.c
index ee58cb92da6b10..102db38f13a9bb 100644
--- a/clang/test/CodeGen/X86/x86-vec-i128.c
+++ b/clang/test/CodeGen/X86/x86-vec-i128.c
@@ -1,16 +1,16 @@
 // RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +sse2 -emit-llvm -o - | FileCheck %s --check-prefixes=CLANG10ABI128,MEM256ALIGN32,MEM512ALIGN64
 // RUN: %clang_cc1 -triple x86_64-netbsd %s -target-feature +sse2 -emit-llvm -o - | FileCheck %s --check-prefixes=CLANG10ABI128,MEM256ALIGN32,MEM512ALIGN64
-// RUN: %clang_cc1 -triple x86_64-apple-darwin %s -target-feature +sse2 -emit-llvm -o - | FileCheck %s --check-prefixes=CLANG9ABI128,MEM256ALIGN16,MEM512ALIGN16
-// RUN: %clang_cc1 -triple x86_64-scei-ps4 %s -target-feature +sse2 -emit-llvm -o - | FileCheck %s --check-prefixes=CLANG9ABI128,MEM256ALIGN32,MEM512ALIGN64
-// RUN: %clang_cc1 -triple x86_64-unknown-freebsd10.0 %s -target-feature +sse2 -emit-llvm -o - | FileCheck %s --check-prefixes=CLANG9ABI128,MEM256ALIGN32,MEM512ALIGN64
-// RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +sse2 -emit-llvm -o - -fclang-abi-compat=9 | FileCheck %s --check-prefixes=CLANG9ABI128,MEM256ALIGN32,MEM512ALIGN64
+// RUN: %clang_cc1 -triple x86_64-apple-darwin %s -target-feature +sse2 -emit-llvm -o - | FileCheck %s --check-prefixes=CLANG9ABI128,LEGACYMEM256ALIGN16,LEGACYMEM512ALIGN16
+// RUN: %clang_cc1 -triple x86_64-scei-ps4 %s -target-feature +sse2 -emit-llvm -o - | FileCheck %s --check-prefixes=CLANG9ABI128,LEGACYMEM256ALIGN32,LEGACYMEM512ALIGN64
+// RUN: %clang_cc1 -triple x86_64-unknown-freebsd10.0 %s -target-feature +sse2 -emit-llvm -o - | FileCheck %s --check-prefixes=CLANG9ABI128,LEGACYMEM256ALIGN32,LEGACYMEM512ALIGN64
+// RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +sse2 -emit-llvm -o - -fclang-abi-compat=9 | FileCheck %s --check-prefixes=CLANG9ABI128,LEGACYMEM256ALIGN32,LEGACYMEM512ALIGN64
 
 // RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +avx -emit-llvm -o - | FileCheck %s --check-prefixes=CLANG10ABI128,CLANG10ABI256,MEM512ALIGN64
 // RUN: %clang_cc1 -triple x86_64-netbsd %s -target-feature +avx -emit-llvm -o - | FileCheck %s --check-prefixes=CLANG10ABI128,CLANG10ABI256,MEM512ALIGN64
-// RUN: %clang_cc1 -triple x86_64-apple-darwin %s -target-feature +avx -emit-llvm -o - | FileCheck %s --check-prefixes=CLANG9ABI128,CLANG9ABI256,MEM512ALIGN32
-// RUN: %clang_cc1 -triple x86_64-scei-ps4 %s -target-feature +avx -emit-llvm -o - | FileCheck %s --check-prefixes=CLANG9ABI128,CLANG9ABI256,MEM512ALIGN64
-// RUN: %clang_cc1 -triple x86_64-unknown-freebsd10.0 %s -target-feature +avx -emit-llvm -o - | FileCheck %s --check-prefixes=CLANG9ABI128,CLANG9ABI256,MEM512ALIGN64
-// RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +avx -emit-llvm -o - -fclang-abi-compat=9 | FileCheck %s --check-prefixes=CLANG9ABI128,CLANG9ABI256,MEM512ALIGN64
+// RUN: %clang_cc1 -triple x86_64-apple-darwin %s -target-feature +avx -emit-llvm -o - | FileCheck %s --check-prefixes=CLANG9ABI128,CLANG9ABI256,LEGACYMEM512ALIGN32
+// RUN: %clang_cc1 -triple x86_64-scei-ps4 %s -target-feature +avx -emit-llvm -o - | FileCheck %s --check-prefixes=CLANG9ABI128,CLANG9ABI256,LEGACYMEM512ALIGN64
+// RUN: %clang_cc1 -triple x86_64-unknown-freebsd10.0 %s -target-feature +avx -emit-llvm -o - | FileCheck %s --check-prefixes=CLANG9ABI128,CLANG9ABI256,LEGACYMEM512ALIGN64
+// RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +avx -emit-llvm -o - -fclang-abi-compat=9 | FileCheck %s --check-prefixes=CLANG9ABI128,CLANG9ABI256,LEGACYMEM512ALIGN64
 
 // RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +avx512f -emit-llvm -o - | FileCheck %s --check-prefixes=CLANG10ABI128,CLANG10ABI256,CLANG10ABI512
 // RUN: %clang_cc1 -triple x86_64-netbsd %s -target-feature +avx512f -emit-llvm -o - | FileCheck %s --check-prefixes=CLANG10ABI128,CLANG10ABI256,CLANG10ABI512
@@ -32,8 +32,9 @@ typedef unsigned long long v32u64 __attribute__((vector_size(32)));
 typedef unsigned __int128 v32u128 __attribute__((vector_size(32)));
 
 v32u64 test_v32u128(v32u64 a, v32u128 b) {
-// MEM256ALIGN16: define{{.*}} <4 x i64> @test_v32u128(ptr noundef byval(<4 x i64>) align 16 %{{.*}}, ptr noundef byval(<2 x i128>) align 16 %{{.*}})
-// MEM256ALIGN32: define{{.*}} <4 x i64> @test_v32u128(ptr noundef byval(<4 x i64>) align 32 %{{.*}}, ptr noundef byval(<2 x i128>) align 32 %{{.*}})
+// LEGACYMEM256ALIGN16: define{{.*}} <4 x i64> @test_v32u128(ptr noundef byval(<4 x i64>) align 16 %{{.*}}, ptr noundef byval(<2 x i128>) align 16 %{{.*}})
+// MEM256ALIGN32: define{{.*}} void @test_v32u128(ptr dead_on_unwind noalias writable sret(<4 x i64>) align 32 %{{.*}}, ptr noundef byval(<4 x i64>) align 32 %{{.*}}, ptr noundef byval(<2 x i128>) align 32 %{{.*}})
+// LEGACYMEM256ALIGN32: define{{.*}} <4 x i64> @test_v32u128(ptr noundef byval(<4 x i64>) align 32 %{{.*}}, ptr noundef byval(<2 x i128>) align 32 %{{.*}})
 // CLANG10ABI256: define{{.*}} <4 x i64> @test_v32u128(<4 x i64> noundef %{{.*}}, ptr noundef byval(<2 x i128>) align 32 %{{.*}})
 // CLANG9ABI256: define{{.*}} <4 x i64> @test_v32u128(<4 x i64> noundef %{{.*}}, <2 x i128> noundef %{{.*}})
   return a + (v32u64)b;
@@ -43,9 +44,10 @@ typedef unsigned long long v64u64 __attribute__((vector_size(64)));
 typedef unsigned __int128 v64u128 __attribute__((vector_size(64)));
 
 v64u64 test_v64u128(v64u64 a, v64u128 b) {
-// MEM512ALIGN16: define{{.*}} <8 x i64> @test_v64u128(ptr noundef byval(<8 x i64>) align 16 %{{.*}}, ptr noundef byval(<4 x i128>) align 16 %{{.*}})
-// MEM512ALIGN32: define{{.*}} <8 x i64> @test_v64u128(ptr noundef byval(<8 x i64>) align 32 %{{.*}}, ptr noundef byval(<4 x i128>) align 32 %{{.*}})
-// MEM512ALIGN64: define{{.*}} <8 x i64> @test_v64u128(ptr noundef byval(<8 x i64>) align 64 %{{.*}}, ptr noundef byval(<4 x i128>) align 64 %{{.*}})
+// LEGACYMEM512ALIGN16: define{{.*}} <8 x i64> @test_v64u128(ptr noundef byval(<8 x i64>) align 16 %{{.*}}, ptr noundef byval(<4 x i128>) align 16 %{{.*}})
+// LEGACYMEM512ALIGN32: define{{.*}} <8 x i64> @test_v64u128(ptr noundef byval(<8 x i64>) align 32 %{{.*}}, ptr noundef byval(<4 x i128>) align 32 %{{.*}})
+// MEM512ALIGN64: define{{.*}} void @test_v64u128(ptr dead_on_unwind noalias writable sret(<8 x i64>) align 64 %{{.*}}, ptr noundef byval(<8 x i64>) align 64 %{{.*}}, ptr noundef byval(<4 x i128>) align 64 %{{.*}})
+// LEGACYMEM512ALIGN64: define{{.*}} <8 x i64> @test_v64u128(ptr noundef byval(<8 x i64>) align 64 %{{.*}}, ptr noundef byval(<4 x i128>) align 64 %{{.*}})
 // CLANG10ABI512: define{{.*}} <8 x i64> @test_v64u128(<8 x i64> noundef %{{.*}}, ptr noundef byval(<4 x i128>) align 64 %{{.*}})
 // CLANG9ABI512: define{{.*}} <8 x i64> @test_v64u128(<8 x i64> noundef %{{.*}}, <4 x i128> noundef %{{.*}})
   return a + (v64u64)b;
diff --git a/clang/test/CodeGen/X86/x86_64-atomic-vector-abi.c b/clang/test/CodeGen/X86/x86_64-atomic-vector-abi.c
new file mode 100644
index 00000000000000..6d0bf403879fb3
--- /dev/null
+++ b/clang/test/CodeGen/X86/x86_64-atomic-vector-abi.c
@@ -0,0 +1,92 @@
+// RUN: %clang_cc1 -triple x86_64-linux-gnu %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -target-feature +avx %s -emit-llvm -o - | FileCheck %s --check-prefix=AVX
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -target-feature +avx -fclang-abi-compat=22 %s -emit-llvm -o - | FileCheck %s --check-prefix=LEGACY
+
+struct F3 {
+  float a[3];
+};
+
+typedef _Atomic(float) atomic_float;
+typedef _Atomic(struct F3) atomic_f3;
+typedef _Float16 v1hf __attribute__((vector_size(2)));
+typedef __bf16 v1bf __attribute__((vector_size(2)));
+typedef float v1sf __attribute__((vector_size(4)));
+typedef double v1df __attribute__((vector_size(8)));
+typedef __int128 v2ti __attribute__((vector_size(32)));
+typedef _Atomic(v1hf) atomic_v1hf;
+typedef _Atomic(v1bf) atomic_v1bf;
+typedef _Atomic(v1sf) atomic_v1sf;
+typedef _Atomic(v1df) atomic_v1df;
+typedef _Atomic(v2ti) atomic_v2ti;
+
+// CHECK-LABEL: define dso_local void @take_float(
+// CHECK-SAME: float %x)
+void take_float(atomic_float x) {}
+
+// CHECK-LABEL: define dso_local float @ret_float(
+// CHECK-SAME: float %x)
+atomic_float ret_float(atomic_float x) {
+  return x;
+}
+
+// CHECK-LABEL: define dso_local void @take_f3(
+// CHECK-SAME: ptr noundef byval({ %struct.F3, [4 x i8] }) align 16 %x)
+void take_f3(atomic_f3 x) {}
+
+// CHECK-LABEL: define dso_local void @ret_f3(
+// CHECK-SAME: ptr {{.*}}sret({ %struct.F3, [4 x i8] }) align 16 %agg.result,
+// CHECK-SAME: ptr noundef byval({ %struct.F3, [4 x i8] }) align 16 %x)
+atomic_f3 ret_f3(atomic_f3 x) {
+  return x;
+}
+
+// AVX-LABEL: define dso_local void @ret_v1hf(
+// AVX-SAME: ptr {{.*}}sret(<1 x half>) align 2 %agg.result,
+// AVX-SAME: ptr noundef byval(<1 x half>) align 8 %{{.*}})
+atomic_v1hf ret_v1hf(atomic_v1hf x) {
+  return x;
+}
+
+// AVX-LABEL: define dso_local void @ret_v1bf(
+// AVX-SAME: ptr {{.*}}sret(<1 x bfloat>) align 2 %agg.result,
+// AVX-SAME: ptr noundef byval(<1 x bfloat>) align 8 %{{.*}})
+atomic_v1bf ret_v1bf(atomic_v1bf x) {
+  return x;
+}
+
+// AVX-LABEL: define dso_local void @take_v1sf(
+// AVX-SAME: ptr noundef byval(<1 x float>) align 8 %{{.*}})
+// LEGACY-LABEL: define dso_local void @take_v1sf(
+// LEGACY-SAME: <1 x float> %x)
+void take_v1sf(atomic_v1sf x) {}
+
+// AVX-LABEL: define dso_local void @ret_v1sf(
+// AVX-SAME: ptr {{.*}}sret(<1 x float>) align 4 %agg.result,
+// AVX-SAME: ptr noundef byval(<1 x float>) align 8 %{{.*}})
+// LEGACY-LABEL: define dso_local <1 x float> @ret_v1sf(
+// LEGACY-SAME: <1 x float> %x)
+atomic_v1sf ret_v1sf(atomic_v1sf x) {
+  return x;
+}
+
+// AVX-LABEL: define dso_local void @ret_v1df(
+// AVX-SAME: ptr {{.*}}sret(<1 x double>) align 8 %agg.result,
+// AVX-SAME: ptr noundef byval(<1 x double>) align 8 %{{.*}})
+atomic_v1df ret_v1df(atomic_v1df x) {
+  return x;
+}
+
+// AVX-LABEL: define dso_local void @take_v2ti(
+// AVX-SAME: ptr noundef byval(<2 x i128>) align 32 %{{.*}})
+// LEGACY-LABEL: define dso_local void @take_v2ti(
+// LEGACY-SAME: <2 x i128> %x)
+void take_v2ti(atomic_v2ti x) {}
+
+// AVX-LABEL: define dso_local void @ret_v2ti(
+// AVX-SAME: ptr {{.*}}sret(<2 x i128>) align 32 %agg.result,
+// AVX-SAME: ptr noundef byval(<2 x i128>) align 32 %{{.*}})
+// LEGACY-LABEL: define dso_local <2 x i128> @ret_v2ti(
+// LEGACY-SAME: <2 x i128> %x)
+atomic_v2ti ret_v2ti(atomic_v2ti x) {
+  return x;
+}
diff --git a/clang/test/CodeGen/builtin-masked.c b/clang/test/CodeGen/builtin-masked.c
index c1ae989d504118..93417017a39396 100644
--- a/clang/test/CodeGen/builtin-masked.c
+++ b/clang/test/CodeGen/builtin-masked.c
@@ -5,8 +5,8 @@ typedef int v8i __attribute__((ext_vector_type(8)));
 typedef _Bool v8b __attribute__((ext_vector_type(8)));
 typedef int gv8i [[gnu::vector_size(sizeof(int) * 8)]];
 
-// CHECK-LABEL: define dso_local <8 x i32> @test_load(
-// CHECK-SAME: i8 noundef [[M_COERCE:%.*]], ptr noundef [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-LABEL: define dso_local void @test_load(
+// CHECK-SAME: ptr dead_on_unwind noalias writable sret(<8 x i32>) align 32 [[AGG_RESULT:%.*]], i8 noundef [[M_COERCE:%.*]], ptr noundef [[P:%.*]]) #[[ATTR0:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[M:%.*]] = alloca i8, align 1
 // CHECK-NEXT:    [[M_ADDR:%.*]] = alloca i8, align 1
@@ -21,14 +21,17 @@ typedef int gv8i [[gnu::vector_size(sizeof(int) * 8)]];
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[LOAD_BITS2]] to <8 x i1>
 // CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[P_ADDR]], align 8
 // CHECK-NEXT:    [[MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 4 [[TMP2]], <8 x i1> [[TMP1]], <8 x i32> poison)
-// CHECK-NEXT:    ret <8 x i32> [[MASKED_LOAD]]
+// CHECK-NEXT:    store <8 x i32> [[MASKED_LOAD]], ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    store <8 x i32> [[TMP3]], ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    ret void
 //
 v8i test_load(v8b m, int *p) {
   return __builtin_masked_load(m, p);
 }
 
-// CHECK-LABEL: define dso_local <8 x i32> @test_load_passthru(
-// CHECK-SAME: i8 noundef [[M_COERCE:%.*]], ptr noundef [[P:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]]) #[[ATTR0]] {
+// CHECK-LABEL: define dso_local void @test_load_passthru(
+// CHECK-SAME: ptr dead_on_unwind noalias writable sret(<8 x i32>) align 32 [[AGG_RESULT:%.*]], i8 noundef [[M_COERCE:%.*]], ptr noundef [[P:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[M:%.*]] = alloca i8, align 1
 // CHECK-NEXT:    [[M_ADDR:%.*]] = alloca i8, align 1
@@ -47,14 +50,17 @@ v8i test_load(v8b m, int *p) {
 // CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[P_ADDR]], align 8
 // CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i32>, ptr [[T_ADDR]], align 32
 // CHECK-NEXT:    [[MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 4 [[TMP3]], <8 x i1> [[TMP2]], <8 x i32> [[TMP4]])
-// CHECK-NEXT:    ret <8 x i32> [[MASKED_LOAD]]
+// CHECK-NEXT:    store <8 x i32> [[MASKED_LOAD]], ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    store <8 x i32> [[TMP5]], ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    ret void
 //
 v8i test_load_passthru(v8b m, int *p, v8i t) {
   return __builtin_masked_load(m, p, t);
 }
 
-// CHECK-LABEL: define dso_local <8 x i32> @test_load_expand(
-// CHECK-SAME: i8 noundef [[M_COERCE:%.*]], ptr noundef [[P:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]]) #[[ATTR0]] {
+// CHECK-LABEL: define dso_local void @test_load_expand(
+// CHECK-SAME: ptr dead_on_unwind noalias writable sret(<8 x i32>) align 32 [[AGG_RESULT:%.*]], i8 noundef [[M_COERCE:%.*]], ptr noundef [[P:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[M:%.*]] = alloca i8, align 1
 // CHECK-NEXT:    [[M_ADDR:%.*]] = alloca i8, align 1
@@ -73,14 +79,17 @@ v8i test_load_passthru(v8b m, int *p, v8i t) {
 // CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[P_ADDR]], align 8
 // CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i32>, ptr [[T_ADDR]], align 32
 // CHECK-NEXT:    [[MASKED_EXPAND_LOAD:%.*]] = call <8 x i32> @llvm.masked.expandload.v8i32.p0(ptr [[TMP3]], <8 x i1> [[TMP2]], <8 x i32> [[TMP4]])
-// CHECK-NEXT:    ret <8 x i32> [[MASKED_EXPAND_LOAD]]
+// CHECK-NEXT:    store <8 x i32> [[MASKED_EXPAND_LOAD]], ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    store <8 x i32> [[TMP5]], ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    ret void
 //
 v8i test_load_expand(v8b m, int *p, v8i t) {
   return __builtin_masked_expand_load(m, p, t);
 }
 
 // CHECK-LABEL: define dso_local void @test_store(
-// CHECK-SAME: i8 noundef [[M_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr noundef [[P:%.*]]) #[[ATTR3:[0-9]+]] {
+// CHECK-SAME: i8 noundef [[M_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr noundef [[P:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[M:%.*]] = alloca i8, align 1
 // CHECK-NEXT:    [[M_ADDR:%.*]] = alloca i8, align 1
@@ -106,7 +115,7 @@ void test_store(v8b m, v8i v, int *p) {
 }
 
 // CHECK-LABEL: define dso_local void @gtest_store(
-// CHECK-SAME: i8 noundef [[M_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr noundef [[P:%.*]]) #[[ATTR3]] {
+// CHECK-SAME: i8 noundef [[M_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr noundef [[P:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[M:%.*]] = alloca i8, align 1
 // CHECK-NEXT:    [[M_ADDR:%.*]] = alloca i8, align 1
@@ -132,7 +141,7 @@ void gtest_store(v8b m, gv8i v, int *p) {
 }
 
 // CHECK-LABEL: define dso_local void @test_compress_store(
-// CHECK-SAME: i8 noundef [[M_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr noundef [[P:%.*]]) #[[ATTR3]] {
+// CHECK-SAME: i8 noundef [[M_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr noundef [[P:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[M:%.*]] = alloca i8, align 1
 // CHECK-NEXT:    [[M_ADDR:%.*]] = alloca i8, align 1
@@ -158,7 +167,7 @@ void test_compress_store(v8b m, v8i v, int *p) {
 }
 
 // CHECK-LABEL: define dso_local void @gtest_compress_store(
-// CHECK-SAME: i8 noundef [[M_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr noundef [[P:%.*]]) #[[ATTR3]] {
+// CHECK-SAME: i8 noundef [[M_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr noundef [[P:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[M:%.*]] = alloca i8, align 1
 // CHECK-NEXT:    [[M_ADDR:%.*]] = alloca i8, align 1
@@ -183,8 +192,8 @@ void gtest_compress_store(v8b m, gv8i v, int *p) {
   __builtin_masked_compress_store(m, v, p);
 }
 
-// CHECK-LABEL: define dso_local <8 x i32> @test_gather(
-// CHECK-SAME: i8 noundef [[MASK_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr noundef [[PTR:%.*]]) #[[ATTR0]] {
+// CHECK-LABEL: define dso_local void @test_gather(
+// CHECK-SAME: ptr dead_on_unwind noalias writable sret(<8 x i32>) align 32 [[AGG_RESULT:%.*]], i8 noundef [[MASK_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr noundef [[PTR:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[MASK:%.*]] = alloca i8, align 1
 // CHECK-NEXT:    [[MASK_ADDR:%.*]] = alloca i8, align 1
@@ -204,14 +213,17 @@ void gtest_compress_store(v8b m, gv8i v, int *p) {
 // CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
 // CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i32, ptr [[TMP4]], <8 x i32> [[TMP3]]
 // CHECK-NEXT:    [[MASKED_GATHER:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> align 4 [[TMP5]], <8 x i1> [[TMP2]], <8 x i32> poison)
-// CHECK-NEXT:    ret <8 x i32> [[MASKED_GATHER]]
+// CHECK-NEXT:    store <8 x i32> [[MASKED_GATHER]], ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i32>, ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    store <8 x i32> [[TMP6]], ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    ret void
 //
 v8i test_gather(v8b mask, v8i idx, int *ptr) {
   return __builtin_masked_gather(mask, idx, ptr);
 }
 
-// CHECK-LABEL: define dso_local <8 x i32> @gtest_gather(
-// CHECK-SAME: i8 noundef [[MASK_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr noundef [[PTR:%.*]]) #[[ATTR0]] {
+// CHECK-LABEL: define dso_local void @gtest_gather(
+// CHECK-SAME: ptr dead_on_unwind noalias writable sret(<8 x i32>) align 32 [[AGG_RESULT:%.*]], i8 noundef [[MASK_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr noundef [[PTR:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[MASK:%.*]] = alloca i8, align 1
 // CHECK-NEXT:    [[MASK_ADDR:%.*]] = alloca i8, align 1
@@ -231,14 +243,17 @@ v8i test_gather(v8b mask, v8i idx, int *ptr) {
 // CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
 // CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i32, ptr [[TMP4]], <8 x i32> [[TMP3]]
 // CHECK-NEXT:    [[MASKED_GATHER:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> align 4 [[TMP5]], <8 x i1> [[TMP2]], <8 x i32> poison)
-// CHECK-NEXT:    ret <8 x i32> [[MASKED_GATHER]]
+// CHECK-NEXT:    store <8 x i32> [[MASKED_GATHER]], ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i32>, ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    store <8 x i32> [[TMP6]], ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    ret void
 //
 v8i gtest_gather(v8b mask, gv8i idx, int *ptr) {
   return __builtin_masked_gather(mask, idx, ptr);
 }
 
 // CHECK-LABEL: define dso_local void @test_scatter(
-// CHECK-SAME: i8 noundef [[MASK_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP1:%.*]], ptr noundef [[PTR:%.*]]) #[[ATTR3]] {
+// CHECK-SAME: i8 noundef [[MASK_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP1:%.*]], ptr noundef [[PTR:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[MASK:%.*]] = alloca i8, align 1
 // CHECK-NEXT:    [[MASK_ADDR:%.*]] = alloca i8, align 1
@@ -269,7 +284,7 @@ void test_scatter(v8b mask, v8i val, v8i idx, int *ptr) {
 }
 
 // CHECK-LABEL: define dso_local void @gtest_scatter(
-// CHECK-SAME: i8 noundef [[MASK_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP1:%.*]], ptr noundef [[PTR:%.*]]) #[[ATTR3]] {
+// CHECK-SAME: i8 noundef [[MASK_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP1:%.*]], ptr noundef [[PTR:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[MASK:%.*]] = alloca i8, align 1
 // CHECK-NEXT:    [[MASK_ADDR:%.*]] = alloca i8, align 1
@@ -299,8 +314,8 @@ void gtest_scatter(v8b mask, gv8i val, gv8i idx, int *ptr) {
   __builtin_masked_scatter(mask, val, idx, ptr);
 }
 
-// CHECK-LABEL: define dso_local <8 x i32> @test_load_as(
-// CHECK-SAME: i8 noundef [[MASK_COERCE:%.*]], ptr addrspace(42) noundef [[PTR:%.*]]) #[[ATTR0]] {
+// CHECK-LABEL: define dso_local void @test_load_as(
+// CHECK-SAME: ptr dead_on_unwind noalias writable sret(<8 x i32>) align 32 [[AGG_RESULT:%.*]], i8 noundef [[MASK_COERCE:%.*]], ptr addrspace(42) noundef [[PTR:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[MASK:%.*]] = alloca i8, align 1
 // CHECK-NEXT:    [[MASK_ADDR:%.*]] = alloca i8, align 1
@@ -315,14 +330,17 @@ void gtest_scatter(v8b mask, gv8i val, gv8i idx, int *ptr) {
 // CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[LOAD_BITS2]] to <8 x i1>
 // CHECK-NEXT:    [[TMP2:%.*]] = load ptr addrspace(42), ptr [[PTR_ADDR]], align 8
 // CHECK-NEXT:    [[MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p42(ptr addrspace(42) align 4 [[TMP2]], <8 x i1> [[TMP1]], <8 x i32> poison)
-// CHECK-NEXT:    ret <8 x i32> [[MASKED_LOAD]]
+// CHECK-NEXT:    store <8 x i32> [[MASKED_LOAD]], ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    store <8 x i32> [[TMP3]], ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    ret void
 //
 v8i test_load_as(v8b mask, int __attribute__((address_space(42))) * ptr) {
   return __builtin_masked_load(mask, ptr);
 }
 
 // CHECK-LABEL: define dso_local void @test_store_as(
-// CHECK-SAME: i8 noundef [[M_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr addrspace(42) noundef [[P:%.*]]) #[[ATTR3]] {
+// CHECK-SAME: i8 noundef [[M_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr addrspace(42) noundef [[P:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[M:%.*]] = alloca i8, align 1
 // CHECK-NEXT:    [[M_ADDR:%.*]] = alloca i8, align 1
@@ -348,7 +366,7 @@ void test_store_as(v8b m, v8i v, int __attribute__((address_space(42))) *p) {
 }
 
 // CHECK-LABEL: define dso_local void @gtest_store_as(
-// CHECK-SAME: i8 noundef [[M_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr addrspace(42) noundef [[P:%.*]]) #[[ATTR3]] {
+// CHECK-SAME: i8 noundef [[M_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr addrspace(42) noundef [[P:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[M:%.*]] = alloca i8, align 1
 // CHECK-NEXT:    [[M_ADDR:%.*]] = alloca i8, align 1
@@ -373,8 +391,8 @@ void gtest_store_as(v8b m, gv8i v, int __attribute__((address_space(42))) *p) {
   __builtin_masked_store(m, v, p);
 }
 
-// CHECK-LABEL: define dso_local <8 x i32> @test_gather_as(
-// CHECK-SAME: i8 noundef [[MASK_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr addrspace(42) noundef [[PTR:%.*]]) #[[ATTR0]] {
+// CHECK-LABEL: define dso_local void @test_gather_as(
+// CHECK-SAME: ptr dead_on_unwind noalias writable sret(<8 x i32>) align 32 [[AGG_RESULT:%.*]], i8 noundef [[MASK_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr addrspace(42) noundef [[PTR:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[MASK:%.*]] = alloca i8, align 1
 // CHECK-NEXT:    [[MASK_ADDR:%.*]] = alloca i8, align 1
@@ -394,14 +412,17 @@ void gtest_store_as(v8b m, gv8i v, int __attribute__((address_space(42))) *p) {
 // CHECK-NEXT:    [[TMP4:%.*]] = load ptr addrspace(42), ptr [[PTR_ADDR]], align 8
 // CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i32, ptr addrspace(42) [[TMP4]], <8 x i32> [[TMP3]]
 // CHECK-NEXT:    [[MASKED_GATHER:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p42(<8 x ptr addrspace(42)> align 4 [[TMP5]], <8 x i1> [[TMP2]], <8 x i32> poison)
-// CHECK-NEXT:    ret <8 x i32> [[MASKED_GATHER]]
+// CHECK-NEXT:    store <8 x i32> [[MASKED_GATHER]], ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i32>, ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    store <8 x i32> [[TMP6]], ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    ret void
 //
 v8i test_gather_as(v8b mask, v8i idx, int __attribute__((address_space(42))) *ptr) {
   return __builtin_masked_gather(mask, idx, ptr);
 }
 
-// CHECK-LABEL: define dso_local <8 x i32> @gtest_gather_as(
-// CHECK-SAME: i8 noundef [[MASK_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr addrspace(42) noundef [[PTR:%.*]]) #[[ATTR0]] {
+// CHECK-LABEL: define dso_local void @gtest_gather_as(
+// CHECK-SAME: ptr dead_on_unwind noalias writable sret(<8 x i32>) align 32 [[AGG_RESULT:%.*]], i8 noundef [[MASK_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr addrspace(42) noundef [[PTR:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[MASK:%.*]] = alloca i8, align 1
 // CHECK-NEXT:    [[MASK_ADDR:%.*]] = alloca i8, align 1
@@ -421,14 +442,17 @@ v8i test_gather_as(v8b mask, v8i idx, int __attribute__((address_space(42))) *pt
 // CHECK-NEXT:    [[TMP4:%.*]] = load ptr addrspace(42), ptr [[PTR_ADDR]], align 8
 // CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i32, ptr addrspace(42) [[TMP4]], <8 x i32> [[TMP3]]
 // CHECK-NEXT:    [[MASKED_GATHER:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p42(<8 x ptr addrspace(42)> align 4 [[TMP5]], <8 x i1> [[TMP2]], <8 x i32> poison)
-// CHECK-NEXT:    ret <8 x i32> [[MASKED_GATHER]]
+// CHECK-NEXT:    store <8 x i32> [[MASKED_GATHER]], ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    [[TMP6:%.*]] = load <8 x i32>, ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    store <8 x i32> [[TMP6]], ptr [[AGG_RESULT]], align 32
+// CHECK-NEXT:    ret void
 //
 v8i gtest_gather_as(v8b mask, gv8i idx, int __attribute__((address_space(42))) *ptr) {
   return __builtin_masked_gather(mask, idx, ptr);
 }
 
 // CHECK-LABEL: define dso_local void @test_scatter_as(
-// CHECK-SAME: i8 noundef [[MASK_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP1:%.*]], ptr addrspace(42) noundef [[PTR:%.*]]) #[[ATTR3]] {
+// CHECK-SAME: i8 noundef [[MASK_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP1:%.*]], ptr addrspace(42) noundef [[PTR:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[MASK:%.*]] = alloca i8, align 1
 // CHECK-NEXT:    [[MASK_ADDR:%.*]] = alloca i8, align 1
@@ -459,7 +483,7 @@ void test_scatter_as(v8b mask, v8i val, v8i idx, int __attribute__((address_spac
 }
 
 // CHECK-LABEL: define dso_local void @gtest_scatter_as(
-// CHECK-SAME: i8 noundef [[MASK_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP1:%.*]], ptr addrspace(42) noundef [[PTR:%.*]]) #[[ATTR3]] {
+// CHECK-SAME: i8 noundef [[MASK_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP1:%.*]], ptr addrspace(42) noundef [[PTR:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[MASK:%.*]] = alloca i8, align 1
 // CHECK-NEXT:    [[MASK_ADDR:%.*]] = alloca i8, align 1
diff --git a/clang/test/CodeGen/builtin-maximumnum-minimumnum.c b/clang/test/CodeGen/builtin-maximumnum-minimumnum.c
index aa18d9ca217f7f..4167a473b0bfac 100644
--- a/clang/test/CodeGen/builtin-maximumnum-minimumnum.c
+++ b/clang/test/CodeGen/builtin-maximumnum-minimumnum.c
@@ -67,8 +67,8 @@ float4 pfmin32(float4 a, float4 b) {
 double2 pfmin64(double2 a, double2 b) {
 	return __builtin_elementwise_minimumnum(a, b);
 }
-// CHECK-LABEL: define dso_local noundef <2 x x86_fp80> @_Z7pfmin80Dv2_eS_(
-// CHECK-SAME: ptr noundef byval(<2 x x86_fp80>) align 32 [[TMP0:%.*]], ptr noundef byval(<2 x x86_fp80>) align 32 [[TMP1:%.*]]) #[[ATTR2:[0-9]+]] {
+// CHECK-LABEL: define dso_local void @_Z7pfmin80Dv2_eS_(
+// CHECK-SAME: ptr dead_on_unwind noalias writable sret(<2 x x86_fp80>) align 32 [[AGG_RESULT:%.*]], ptr noundef byval(<2 x x86_fp80>) align 32 [[TMP0:%.*]], ptr noundef byval(<2 x x86_fp80>) align 32 [[TMP1:%.*]]) #[[ATTR2:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca <2 x x86_fp80>, align 32
 // CHECK-NEXT:    [[B_ADDR:%.*]] = alloca <2 x x86_fp80>, align 32
@@ -79,7 +79,10 @@ double2 pfmin64(double2 a, double2 b) {
 // CHECK-NEXT:    [[TMP2:%.*]] = load <2 x x86_fp80>, ptr [[A_ADDR]], align 32, !tbaa [[TBAA6]]
 // CHECK-NEXT:    [[TMP3:%.*]] = load <2 x x86_fp80>, ptr [[B_ADDR]], align 32, !tbaa [[TBAA6]]
 // CHECK-NEXT:    [[ELT_MINIMUMNUM:%.*]] = call <2 x x86_fp80> @llvm.minimumnum.v2f80(<2 x x86_fp80> [[TMP2]], <2 x x86_fp80> [[TMP3]])
-// CHECK-NEXT:    ret <2 x x86_fp80> [[ELT_MINIMUMNUM]]
+// CHECK-NEXT:    store <2 x x86_fp80> [[ELT_MINIMUMNUM]], ptr [[AGG_RESULT]], align 32, !tbaa [[TBAA6]]
+// CHECK-NEXT:    [[TMP4:%.*]] = load <2 x x86_fp80>, ptr [[AGG_RESULT]], align 32, !tbaa [[TBAA6]]
+// CHECK-NEXT:    store <2 x x86_fp80> [[TMP4]], ptr [[AGG_RESULT]], align 32, !tbaa [[TBAA6]]
+// CHECK-NEXT:    ret void
 //
 ldouble2 pfmin80(ldouble2 a, ldouble2 b) {
 	return __builtin_elementwise_minimumnum(a, b);
@@ -146,8 +149,8 @@ double2 pfmax64(double2 a, double2 b) {
 	return __builtin_elementwise_maximumnum(a, b);
 }
 
-// CHECK-LABEL: define dso_local noundef <2 x x86_fp80> @_Z7pfmax80Dv2_eS_(
-// CHECK-SAME: ptr noundef byval(<2 x x86_fp80>) align 32 [[TMP0:%.*]], ptr noundef byval(<2 x x86_fp80>) align 32 [[TMP1:%.*]]) #[[ATTR2]] {
+// CHECK-LABEL: define dso_local void @_Z7pfmax80Dv2_eS_(
+// CHECK-SAME: ptr dead_on_unwind noalias writable sret(<2 x x86_fp80>) align 32 [[AGG_RESULT:%.*]], ptr noundef byval(<2 x x86_fp80>) align 32 [[TMP0:%.*]], ptr noundef byval(<2 x x86_fp80>) align 32 [[TMP1:%.*]]) #[[ATTR2]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca <2 x x86_fp80>, align 32
 // CHECK-NEXT:    [[B_ADDR:%.*]] = alloca <2 x x86_fp80>, align 32
@@ -158,7 +161,10 @@ double2 pfmax64(double2 a, double2 b) {
 // CHECK-NEXT:    [[TMP2:%.*]] = load <2 x x86_fp80>, ptr [[A_ADDR]], align 32, !tbaa [[TBAA6]]
 // CHECK-NEXT:    [[TMP3:%.*]] = load <2 x x86_fp80>, ptr [[B_ADDR]], align 32, !tbaa [[TBAA6]]
 // CHECK-NEXT:    [[ELT_MINIMUMNUM:%.*]] = call <2 x x86_fp80> @llvm.minimumnum.v2f80(<2 x x86_fp80> [[TMP2]], <2 x x86_fp80> [[TMP3]])
-// CHECK-NEXT:    ret <2 x x86_fp80> [[ELT_MINIMUMNUM]]
+// CHECK-NEXT:    store <2 x x86_fp80> [[ELT_MINIMUMNUM]], ptr [[AGG_RESULT]], align 32, !tbaa [[TBAA6]]
+// CHECK-NEXT:    [[TMP4:%.*]] = load <2 x x86_fp80>, ptr [[AGG_RESULT]], align 32, !tbaa [[TBAA6]]
+// CHECK-NEXT:    store <2 x x86_fp80> [[TMP4]], ptr [[AGG_RESULT]], align 32, !tbaa [[TBAA6]]
+// CHECK-NEXT:    ret void
 //
 ldouble2 pfmax80(ldouble2 a, ldouble2 b) {
 	return __builtin_elementwise_minimumnum(a, b);
diff --git a/clang/test/CodeGen/builtin-maxnum-minnum.c b/clang/test/CodeGen/builtin-maxnum-minnum.c
index d05d43c23bf27f..e277852f51a090 100644
--- a/clang/test/CodeGen/builtin-maxnum-minnum.c
+++ b/clang/test/CodeGen/builtin-maxnum-minnum.c
@@ -67,8 +67,8 @@ float4 pfmin32(float4 a, float4 b) {
 double2 pfmin64(double2 a, double2 b) {
 	return __builtin_elementwise_minnum(a, b);
 }
-// CHECK-LABEL: define dso_local noundef <2 x x86_fp80> @_Z7pfmin80Dv2_eS_(
-// CHECK-SAME: ptr noundef byval(<2 x x86_fp80>) align 32 [[TMP0:%.*]], ptr noundef byval(<2 x x86_fp80>) align 32 [[TMP1:%.*]]) #[[ATTR2:[0-9]+]] {
+// CHECK-LABEL: define dso_local void @_Z7pfmin80Dv2_eS_(
+// CHECK-SAME: ptr dead_on_unwind noalias writable sret(<2 x x86_fp80>) align 32 [[AGG_RESULT:%.*]], ptr noundef byval(<2 x x86_fp80>) align 32 [[TMP0:%.*]], ptr noundef byval(<2 x x86_fp80>) align 32 [[TMP1:%.*]]) #[[ATTR2:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca <2 x x86_fp80>, align 32
 // CHECK-NEXT:    [[B_ADDR:%.*]] = alloca <2 x x86_fp80>, align 32
@@ -79,7 +79,10 @@ double2 pfmin64(double2 a, double2 b) {
 // CHECK-NEXT:    [[TMP2:%.*]] = load <2 x x86_fp80>, ptr [[A_ADDR]], align 32, !tbaa [[CHAR_TBAA6]]
 // CHECK-NEXT:    [[TMP3:%.*]] = load <2 x x86_fp80>, ptr [[B_ADDR]], align 32, !tbaa [[CHAR_TBAA6]]
 // CHECK-NEXT:    [[ELT_MINNUM:%.*]] = call <2 x x86_fp80> @llvm.minnum.v2f80(<2 x x86_fp80> [[TMP2]], <2 x x86_fp80> [[TMP3]])
-// CHECK-NEXT:    ret <2 x x86_fp80> [[ELT_MINNUM]]
+// CHECK-NEXT:    store <2 x x86_fp80> [[ELT_MINNUM]], ptr [[AGG_RESULT]], align 32, !tbaa [[CHAR_TBAA6]]
+// CHECK-NEXT:    [[TMP4:%.*]] = load <2 x x86_fp80>, ptr [[AGG_RESULT]], align 32, !tbaa [[CHAR_TBAA6]]
+// CHECK-NEXT:    store <2 x x86_fp80> [[TMP4]], ptr [[AGG_RESULT]], align 32, !tbaa [[CHAR_TBAA6]]
+// CHECK-NEXT:    ret void
 //
 ldouble2 pfmin80(ldouble2 a, ldouble2 b) {
 	return __builtin_elementwise_minnum(a, b);
@@ -146,8 +149,8 @@ double2 pfmax64(double2 a, double2 b) {
 	return __builtin_elementwise_maxnum(a, b);
 }
 
-// CHECK-LABEL: define dso_local noundef <2 x x86_fp80> @_Z7pfmax80Dv2_eS_(
-// CHECK-SAME: ptr noundef byval(<2 x x86_fp80>) align 32 [[TMP0:%.*]], ptr noundef byval(<2 x x86_fp80>) align 32 [[TMP1:%.*]]) #[[ATTR2]] {
+// CHECK-LABEL: define dso_local void @_Z7pfmax80Dv2_eS_(
+// CHECK-SAME: ptr dead_on_unwind noalias writable sret(<2 x x86_fp80>) align 32 [[AGG_RESULT:%.*]], ptr noundef byval(<2 x x86_fp80>) align 32 [[TMP0:%.*]], ptr noundef byval(<2 x x86_fp80>) align 32 [[TMP1:%.*]]) #[[ATTR2]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca <2 x x86_fp80>, align 32
 // CHECK-NEXT:    [[B_ADDR:%.*]] = alloca <2 x x86_fp80>, align 32
@@ -158,7 +161,10 @@ double2 pfmax64(double2 a, double2 b) {
 // CHECK-NEXT:    [[TMP2:%.*]] = load <2 x x86_fp80>, ptr [[A_ADDR]], align 32, !tbaa [[CHAR_TBAA6]]
 // CHECK-NEXT:    [[TMP3:%.*]] = load <2 x x86_fp80>, ptr [[B_ADDR]], align 32, !tbaa [[CHAR_TBAA6]]
 // CHECK-NEXT:    [[ELT_MINNUM:%.*]] = call <2 x x86_fp80> @llvm.minnum.v2f80(<2 x x86_fp80> [[TMP2]], <2 x x86_fp80> [[TMP3]])
-// CHECK-NEXT:    ret <2 x x86_fp80> [[ELT_MINNUM]]
+// CHECK-NEXT:    store <2 x x86_fp80> [[ELT_MINNUM]], ptr [[AGG_RESULT]], align 32, !tbaa [[CHAR_TBAA6]]
+// CHECK-NEXT:    [[TMP4:%.*]] = load <2 x x86_fp80>, ptr [[AGG_RESULT]], align 32, !tbaa [[CHAR_TBAA6]]
+// CHECK-NEXT:    store <2 x x86_fp80> [[TMP4]], ptr [[AGG_RESULT]], align 32, !tbaa [[CHAR_TBAA6]]
+// CHECK-NEXT:    ret void
 //
 ldouble2 pfmax80(ldouble2 a, ldouble2 b) {
 	return __builtin_elementwise_minnum(a, b);
diff --git a/clang/test/CodeGen/nofpclass.c b/clang/test/CodeGen/nofpclass.c
index eb6932f1f67083..6a69b49693902c 100644
--- a/clang/test/CodeGen/nofpclass.c
+++ b/clang/test/CodeGen/nofpclass.c
@@ -213,112 +213,94 @@ float call_extern_func(float a, double b, _Float16 c) {
 
 // CFINITEONLY: Function Attrs: noinline nounwind optnone
 // CFINITEONLY-LABEL: define dso_local nofpclass(nan inf) double @call_extern_func_vec
-// CFINITEONLY-SAME: (double noundef nofpclass(nan inf) [[A_COERCE:%.*]], <2 x double> noundef nofpclass(nan inf) [[B:%.*]], i32 noundef [[C_COERCE:%.*]]) #[[ATTR2]] {
+// CFINITEONLY-SAME: (double noundef nofpclass(nan inf) [[A_COERCE:%.*]], <2 x double> noundef nofpclass(nan inf) [[B:%.*]], <2 x half> noundef nofpclass(nan inf) [[C:%.*]]) #[[ATTR2]] {
 // CFINITEONLY-NEXT:  entry:
 // CFINITEONLY-NEXT:    [[RETVAL:%.*]] = alloca <2 x float>, align 8
 // CFINITEONLY-NEXT:    [[A:%.*]] = alloca <2 x float>, align 8
-// CFINITEONLY-NEXT:    [[C:%.*]] = alloca <2 x half>, align 4
 // CFINITEONLY-NEXT:    [[A_ADDR:%.*]] = alloca <2 x float>, align 8
 // CFINITEONLY-NEXT:    [[B_ADDR:%.*]] = alloca <2 x double>, align 16
 // CFINITEONLY-NEXT:    [[C_ADDR:%.*]] = alloca <2 x half>, align 4
 // CFINITEONLY-NEXT:    [[COERCE:%.*]] = alloca <2 x float>, align 8
-// CFINITEONLY-NEXT:    [[COERCE3:%.*]] = alloca <2 x half>, align 4
-// CFINITEONLY-NEXT:    [[COERCE4:%.*]] = alloca <2 x float>, align 8
+// CFINITEONLY-NEXT:    [[COERCE2:%.*]] = alloca <2 x float>, align 8
 // CFINITEONLY-NEXT:    store double [[A_COERCE]], ptr [[A]], align 8
 // CFINITEONLY-NEXT:    [[A1:%.*]] = load <2 x float>, ptr [[A]], align 8
-// CFINITEONLY-NEXT:    store i32 [[C_COERCE]], ptr [[C]], align 4
-// CFINITEONLY-NEXT:    [[C2:%.*]] = load <2 x half>, ptr [[C]], align 4
 // CFINITEONLY-NEXT:    store <2 x float> [[A1]], ptr [[A_ADDR]], align 8
 // CFINITEONLY-NEXT:    store <2 x double> [[B]], ptr [[B_ADDR]], align 16
-// CFINITEONLY-NEXT:    store <2 x half> [[C2]], ptr [[C_ADDR]], align 4
+// CFINITEONLY-NEXT:    store <2 x half> [[C]], ptr [[C_ADDR]], align 4
 // CFINITEONLY-NEXT:    [[TMP0:%.*]] = load <2 x float>, ptr [[A_ADDR]], align 8
 // CFINITEONLY-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[B_ADDR]], align 16
 // CFINITEONLY-NEXT:    [[TMP2:%.*]] = load <2 x half>, ptr [[C_ADDR]], align 4
 // CFINITEONLY-NEXT:    store <2 x float> [[TMP0]], ptr [[COERCE]], align 8
 // CFINITEONLY-NEXT:    [[TMP3:%.*]] = load double, ptr [[COERCE]], align 8
-// CFINITEONLY-NEXT:    store <2 x half> [[TMP2]], ptr [[COERCE3]], align 4
-// CFINITEONLY-NEXT:    [[TMP4:%.*]] = load i32, ptr [[COERCE3]], align 4
-// CFINITEONLY-NEXT:    [[CALL:%.*]] = call nnan ninf nofpclass(nan inf) double @extern_func_vec(double noundef nofpclass(nan inf) [[TMP3]], <2 x double> noundef nofpclass(nan inf) [[TMP1]], i32 noundef [[TMP4]])
-// CFINITEONLY-NEXT:    store double [[CALL]], ptr [[COERCE4]], align 8
-// CFINITEONLY-NEXT:    [[TMP5:%.*]] = load <2 x float>, ptr [[COERCE4]], align 8
-// CFINITEONLY-NEXT:    store <2 x float> [[TMP5]], ptr [[RETVAL]], align 8
-// CFINITEONLY-NEXT:    [[TMP6:%.*]] = load double, ptr [[RETVAL]], align 8
-// CFINITEONLY-NEXT:    ret double [[TMP6]]
+// CFINITEONLY-NEXT:    [[CALL:%.*]] = call nnan ninf nofpclass(nan inf) double @extern_func_vec(double noundef nofpclass(nan inf) [[TMP3]], <2 x double> noundef nofpclass(nan inf) [[TMP1]], <2 x half> noundef nofpclass(nan inf) [[TMP2]])
+// CFINITEONLY-NEXT:    store double [[CALL]], ptr [[COERCE2]], align 8
+// CFINITEONLY-NEXT:    [[TMP4:%.*]] = load <2 x float>, ptr [[COERCE2]], align 8
+// CFINITEONLY-NEXT:    store <2 x float> [[TMP4]], ptr [[RETVAL]], align 8
+// CFINITEONLY-NEXT:    [[TMP5:%.*]] = load double, ptr [[RETVAL]], align 8
+// CFINITEONLY-NEXT:    ret double [[TMP5]]
 //
 // CLFINITEONLY: Function Attrs: convergent norecurse nounwind
 // CLFINITEONLY-LABEL: define dso_local nofpclass(nan inf) double @call_extern_func_vec
-// CLFINITEONLY-SAME: (double noundef nofpclass(nan inf) [[A_COERCE:%.*]], <2 x double> noundef nofpclass(nan inf) [[B:%.*]], i32 noundef [[C_COERCE:%.*]]) local_unnamed_addr #[[ATTR5:[0-9]+]] {
+// CLFINITEONLY-SAME: (double noundef nofpclass(nan inf) [[A_COERCE:%.*]], <2 x double> noundef nofpclass(nan inf) [[B:%.*]], <2 x half> noundef nofpclass(nan inf) [[C:%.*]]) local_unnamed_addr #[[ATTR5:[0-9]+]] {
 // CLFINITEONLY-NEXT:  entry:
-// CLFINITEONLY-NEXT:    [[CALL:%.*]] = tail call nnan ninf nofpclass(nan inf) double @extern_func_vec(double noundef nofpclass(nan inf) [[A_COERCE]], <2 x double> noundef nofpclass(nan inf) [[B]], i32 noundef [[C_COERCE]]) #[[ATTR11]]
+// CLFINITEONLY-NEXT:    [[CALL:%.*]] = tail call nnan ninf nofpclass(nan inf) double @extern_func_vec(double noundef nofpclass(nan inf) [[A_COERCE]], <2 x double> noundef nofpclass(nan inf) [[B]], <2 x half> noundef nofpclass(nan inf) [[C]]) #[[ATTR11]]
 // CLFINITEONLY-NEXT:    ret double [[CALL]]
 //
 // NONANS: Function Attrs: noinline nounwind optnone
 // NONANS-LABEL: define dso_local nofpclass(nan) double @call_extern_func_vec
-// NONANS-SAME: (double noundef nofpclass(nan) [[A_COERCE:%.*]], <2 x double> noundef nofpclass(nan) [[B:%.*]], i32 noundef [[C_COERCE:%.*]]) #[[ATTR2]] {
+// NONANS-SAME: (double noundef nofpclass(nan) [[A_COERCE:%.*]], <2 x double> noundef nofpclass(nan) [[B:%.*]], <2 x half> noundef nofpclass(nan) [[C:%.*]]) #[[ATTR2]] {
 // NONANS-NEXT:  entry:
 // NONANS-NEXT:    [[RETVAL:%.*]] = alloca <2 x float>, align 8
 // NONANS-NEXT:    [[A:%.*]] = alloca <2 x float>, align 8
-// NONANS-NEXT:    [[C:%.*]] = alloca <2 x half>, align 4
 // NONANS-NEXT:    [[A_ADDR:%.*]] = alloca <2 x float>, align 8
 // NONANS-NEXT:    [[B_ADDR:%.*]] = alloca <2 x double>, align 16
 // NONANS-NEXT:    [[C_ADDR:%.*]] = alloca <2 x half>, align 4
 // NONANS-NEXT:    [[COERCE:%.*]] = alloca <2 x float>, align 8
-// NONANS-NEXT:    [[COERCE3:%.*]] = alloca <2 x half>, align 4
-// NONANS-NEXT:    [[COERCE4:%.*]] = alloca <2 x float>, align 8
+// NONANS-NEXT:    [[COERCE2:%.*]] = alloca <2 x float>, align 8
 // NONANS-NEXT:    store double [[A_COERCE]], ptr [[A]], align 8
 // NONANS-NEXT:    [[A1:%.*]] = load <2 x float>, ptr [[A]], align 8
-// NONANS-NEXT:    store i32 [[C_COERCE]], ptr [[C]], align 4
-// NONANS-NEXT:    [[C2:%.*]] = load <2 x half>, ptr [[C]], align 4
 // NONANS-NEXT:    store <2 x float> [[A1]], ptr [[A_ADDR]], align 8
 // NONANS-NEXT:    store <2 x double> [[B]], ptr [[B_ADDR]], align 16
-// NONANS-NEXT:    store <2 x half> [[C2]], ptr [[C_ADDR]], align 4
+// NONANS-NEXT:    store <2 x half> [[C]], ptr [[C_ADDR]], align 4
 // NONANS-NEXT:    [[TMP0:%.*]] = load <2 x float>, ptr [[A_ADDR]], align 8
 // NONANS-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[B_ADDR]], align 16
 // NONANS-NEXT:    [[TMP2:%.*]] = load <2 x half>, ptr [[C_ADDR]], align 4
 // NONANS-NEXT:    store <2 x float> [[TMP0]], ptr [[COERCE]], align 8
 // NONANS-NEXT:    [[TMP3:%.*]] = load double, ptr [[COERCE]], align 8
-// NONANS-NEXT:    store <2 x half> [[TMP2]], ptr [[COERCE3]], align 4
-// NONANS-NEXT:    [[TMP4:%.*]] = load i32, ptr [[COERCE3]], align 4
-// NONANS-NEXT:    [[CALL:%.*]] = call nnan nofpclass(nan) double @extern_func_vec(double noundef nofpclass(nan) [[TMP3]], <2 x double> noundef nofpclass(nan) [[TMP1]], i32 noundef [[TMP4]])
-// NONANS-NEXT:    store double [[CALL]], ptr [[COERCE4]], align 8
-// NONANS-NEXT:    [[TMP5:%.*]] = load <2 x float>, ptr [[COERCE4]], align 8
-// NONANS-NEXT:    store <2 x float> [[TMP5]], ptr [[RETVAL]], align 8
-// NONANS-NEXT:    [[TMP6:%.*]] = load double, ptr [[RETVAL]], align 8
-// NONANS-NEXT:    ret double [[TMP6]]
+// NONANS-NEXT:    [[CALL:%.*]] = call nnan nofpclass(nan) double @extern_func_vec(double noundef nofpclass(nan) [[TMP3]], <2 x double> noundef nofpclass(nan) [[TMP1]], <2 x half> noundef nofpclass(nan) [[TMP2]])
+// NONANS-NEXT:    store double [[CALL]], ptr [[COERCE2]], align 8
+// NONANS-NEXT:    [[TMP4:%.*]] = load <2 x float>, ptr [[COERCE2]], align 8
+// NONANS-NEXT:    store <2 x float> [[TMP4]], ptr [[RETVAL]], align 8
+// NONANS-NEXT:    [[TMP5:%.*]] = load double, ptr [[RETVAL]], align 8
+// NONANS-NEXT:    ret double [[TMP5]]
 //
 // NOINFS: Function Attrs: noinline nounwind optnone
 // NOINFS-LABEL: define dso_local nofpclass(inf) double @call_extern_func_vec
-// NOINFS-SAME: (double noundef nofpclass(inf) [[A_COERCE:%.*]], <2 x double> noundef nofpclass(inf) [[B:%.*]], i32 noundef [[C_COERCE:%.*]]) #[[ATTR2]] {
+// NOINFS-SAME: (double noundef nofpclass(inf) [[A_COERCE:%.*]], <2 x double> noundef nofpclass(inf) [[B:%.*]], <2 x half> noundef nofpclass(inf) [[C:%.*]]) #[[ATTR2]] {
 // NOINFS-NEXT:  entry:
 // NOINFS-NEXT:    [[RETVAL:%.*]] = alloca <2 x float>, align 8
 // NOINFS-NEXT:    [[A:%.*]] = alloca <2 x float>, align 8
-// NOINFS-NEXT:    [[C:%.*]] = alloca <2 x half>, align 4
 // NOINFS-NEXT:    [[A_ADDR:%.*]] = alloca <2 x float>, align 8
 // NOINFS-NEXT:    [[B_ADDR:%.*]] = alloca <2 x double>, align 16
 // NOINFS-NEXT:    [[C_ADDR:%.*]] = alloca <2 x half>, align 4
 // NOINFS-NEXT:    [[COERCE:%.*]] = alloca <2 x float>, align 8
-// NOINFS-NEXT:    [[COERCE3:%.*]] = alloca <2 x half>, align 4
-// NOINFS-NEXT:    [[COERCE4:%.*]] = alloca <2 x float>, align 8
+// NOINFS-NEXT:    [[COERCE2:%.*]] = alloca <2 x float>, align 8
 // NOINFS-NEXT:    store double [[A_COERCE]], ptr [[A]], align 8
 // NOINFS-NEXT:    [[A1:%.*]] = load <2 x float>, ptr [[A]], align 8
-// NOINFS-NEXT:    store i32 [[C_COERCE]], ptr [[C]], align 4
-// NOINFS-NEXT:    [[C2:%.*]] = load <2 x half>, ptr [[C]], align 4
 // NOINFS-NEXT:    store <2 x float> [[A1]], ptr [[A_ADDR]], align 8
 // NOINFS-NEXT:    store <2 x double> [[B]], ptr [[B_ADDR]], align 16
-// NOINFS-NEXT:    store <2 x half> [[C2]], ptr [[C_ADDR]], align 4
+// NOINFS-NEXT:    store <2 x half> [[C]], ptr [[C_ADDR]], align 4
 // NOINFS-NEXT:    [[TMP0:%.*]] = load <2 x float>, ptr [[A_ADDR]], align 8
 // NOINFS-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[B_ADDR]], align 16
 // NOINFS-NEXT:    [[TMP2:%.*]] = load <2 x half>, ptr [[C_ADDR]], align 4
 // NOINFS-NEXT:    store <2 x float> [[TMP0]], ptr [[COERCE]], align 8
 // NOINFS-NEXT:    [[TMP3:%.*]] = load double, ptr [[COERCE]], align 8
-// NOINFS-NEXT:    store <2 x half> [[TMP2]], ptr [[COERCE3]], align 4
-// NOINFS-NEXT:    [[TMP4:%.*]] = load i32, ptr [[COERCE3]], align 4
-// NOINFS-NEXT:    [[CALL:%.*]] = call ninf nofpclass(inf) double @extern_func_vec(double noundef nofpclass(inf) [[TMP3]], <2 x double> noundef nofpclass(inf) [[TMP1]], i32 noundef [[TMP4]])
-// NOINFS-NEXT:    store double [[CALL]], ptr [[COERCE4]], align 8
-// NOINFS-NEXT:    [[TMP5:%.*]] = load <2 x float>, ptr [[COERCE4]], align 8
-// NOINFS-NEXT:    store <2 x float> [[TMP5]], ptr [[RETVAL]], align 8
-// NOINFS-NEXT:    [[TMP6:%.*]] = load double, ptr [[RETVAL]], align 8
-// NOINFS-NEXT:    ret double [[TMP6]]
+// NOINFS-NEXT:    [[CALL:%.*]] = call ninf nofpclass(inf) double @extern_func_vec(double noundef nofpclass(inf) [[TMP3]], <2 x double> noundef nofpclass(inf) [[TMP1]], <2 x half> noundef nofpclass(inf) [[TMP2]])
+// NOINFS-NEXT:    store double [[CALL]], ptr [[COERCE2]], align 8
+// NOINFS-NEXT:    [[TMP4:%.*]] = load <2 x float>, ptr [[COERCE2]], align 8
+// NOINFS-NEXT:    store <2 x float> [[TMP4]], ptr [[RETVAL]], align 8
+// NOINFS-NEXT:    [[TMP5:%.*]] = load double, ptr [[RETVAL]], align 8
+// NOINFS-NEXT:    ret double [[TMP5]]
 //
 float2 call_extern_func_vec(float2 a, double2 b, half2 c) {
   return extern_func_vec(a, b, c);
@@ -845,12 +827,11 @@ _Complex _Float16 defined_complex_func_f16_ret(_Complex _Float16 c) {
 
 // CFINITEONLY: Function Attrs: noinline nounwind optnone
 // CFINITEONLY-LABEL: define dso_local nofpclass(nan inf) float @call_variadic
-// CFINITEONLY-SAME: (float noundef nofpclass(nan inf) [[F32:%.*]], double noundef nofpclass(nan inf) [[F64:%.*]], half noundef nofpclass(nan inf) [[F16:%.*]], double noundef nofpclass(nan inf) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(nan inf) [[V2F64:%.*]], i32 noundef [[V2F16_COERCE:%.*]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE:%.*]], double noundef nofpclass(nan inf) [[CF64_COERCE0:%.*]], double noundef nofpclass(nan inf) [[CF64_COERCE1:%.*]], ptr noundef byval({ half, half }) align 8 [[CF16:%.*]]) #[[ATTR2]] {
+// CFINITEONLY-SAME: (float noundef nofpclass(nan inf) [[F32:%.*]], double noundef nofpclass(nan inf) [[F64:%.*]], half noundef nofpclass(nan inf) [[F16:%.*]], double noundef nofpclass(nan inf) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(nan inf) [[V2F64:%.*]], <2 x half> noundef nofpclass(nan inf) [[V2F16:%.*]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE:%.*]], ptr noundef byval({ double, double }) align 8 [[CF64:%.*]], <2 x half> noundef nofpclass(nan inf) [[CF16_COERCE:%.*]]) #[[ATTR2]] {
 // CFINITEONLY-NEXT:  entry:
 // CFINITEONLY-NEXT:    [[V2F32:%.*]] = alloca <2 x float>, align 8
-// CFINITEONLY-NEXT:    [[V2F16:%.*]] = alloca <2 x half>, align 4
 // CFINITEONLY-NEXT:    [[CF32:%.*]] = alloca { float, float }, align 4
-// CFINITEONLY-NEXT:    [[CF64:%.*]] = alloca { double, double }, align 8
+// CFINITEONLY-NEXT:    [[CF16:%.*]] = alloca { half, half }, align 2
 // CFINITEONLY-NEXT:    [[F32_ADDR:%.*]] = alloca float, align 4
 // CFINITEONLY-NEXT:    [[F64_ADDR:%.*]] = alloca double, align 8
 // CFINITEONLY-NEXT:    [[F16_ADDR:%.*]] = alloca half, align 2
@@ -858,33 +839,27 @@ _Complex _Float16 defined_complex_func_f16_ret(_Complex _Float16 c) {
 // CFINITEONLY-NEXT:    [[V2F64_ADDR:%.*]] = alloca <2 x double>, align 16
 // CFINITEONLY-NEXT:    [[V2F16_ADDR:%.*]] = alloca <2 x half>, align 4
 // CFINITEONLY-NEXT:    [[COERCE:%.*]] = alloca <2 x float>, align 8
-// CFINITEONLY-NEXT:    [[COERCE3:%.*]] = alloca <2 x half>, align 4
-// CFINITEONLY-NEXT:    [[COERCE4:%.*]] = alloca { float, float }, align 4
+// CFINITEONLY-NEXT:    [[COERCE2:%.*]] = alloca { float, float }, align 4
 // CFINITEONLY-NEXT:    [[BYVAL_TEMP:%.*]] = alloca { double, double }, align 8
-// CFINITEONLY-NEXT:    [[COERCE5:%.*]] = alloca { half, half }, align 2
+// CFINITEONLY-NEXT:    [[BYVAL_TEMP3:%.*]] = alloca { half, half }, align 8
 // CFINITEONLY-NEXT:    store double [[V2F32_COERCE]], ptr [[V2F32]], align 8
 // CFINITEONLY-NEXT:    [[V2F321:%.*]] = load <2 x float>, ptr [[V2F32]], align 8
-// CFINITEONLY-NEXT:    store i32 [[V2F16_COERCE]], ptr [[V2F16]], align 4
-// CFINITEONLY-NEXT:    [[V2F162:%.*]] = load <2 x half>, ptr [[V2F16]], align 4
 // CFINITEONLY-NEXT:    store <2 x float> [[CF32_COERCE]], ptr [[CF32]], align 4
-// CFINITEONLY-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[CF64]], i32 0, i32 0
-// CFINITEONLY-NEXT:    store double [[CF64_COERCE0]], ptr [[TMP0]], align 8
-// CFINITEONLY-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[CF64]], i32 0, i32 1
-// CFINITEONLY-NEXT:    store double [[CF64_COERCE1]], ptr [[TMP1]], align 8
+// CFINITEONLY-NEXT:    store <2 x half> [[CF16_COERCE]], ptr [[CF16]], align 2
 // CFINITEONLY-NEXT:    store float [[F32]], ptr [[F32_ADDR]], align 4
 // CFINITEONLY-NEXT:    store double [[F64]], ptr [[F64_ADDR]], align 8
 // CFINITEONLY-NEXT:    store half [[F16]], ptr [[F16_ADDR]], align 2
 // CFINITEONLY-NEXT:    store <2 x float> [[V2F321]], ptr [[V2F32_ADDR]], align 8
 // CFINITEONLY-NEXT:    store <2 x double> [[V2F64]], ptr [[V2F64_ADDR]], align 16
-// CFINITEONLY-NEXT:    store <2 x half> [[V2F162]], ptr [[V2F16_ADDR]], align 4
-// CFINITEONLY-NEXT:    [[TMP2:%.*]] = load float, ptr [[F32_ADDR]], align 4
-// CFINITEONLY-NEXT:    [[TMP3:%.*]] = load float, ptr [[F32_ADDR]], align 4
-// CFINITEONLY-NEXT:    [[CONV:%.*]] = fpext nnan ninf float [[TMP3]] to double
-// CFINITEONLY-NEXT:    [[TMP4:%.*]] = load double, ptr [[F64_ADDR]], align 8
-// CFINITEONLY-NEXT:    [[TMP5:%.*]] = load half, ptr [[F16_ADDR]], align 2
-// CFINITEONLY-NEXT:    [[TMP6:%.*]] = load <2 x float>, ptr [[V2F32_ADDR]], align 8
-// CFINITEONLY-NEXT:    [[TMP7:%.*]] = load <2 x double>, ptr [[V2F64_ADDR]], align 16
-// CFINITEONLY-NEXT:    [[TMP8:%.*]] = load <2 x half>, ptr [[V2F16_ADDR]], align 4
+// CFINITEONLY-NEXT:    store <2 x half> [[V2F16]], ptr [[V2F16_ADDR]], align 4
+// CFINITEONLY-NEXT:    [[TMP0:%.*]] = load float, ptr [[F32_ADDR]], align 4
+// CFINITEONLY-NEXT:    [[TMP1:%.*]] = load float, ptr [[F32_ADDR]], align 4
+// CFINITEONLY-NEXT:    [[CONV:%.*]] = fpext nnan ninf float [[TMP1]] to double
+// CFINITEONLY-NEXT:    [[TMP2:%.*]] = load double, ptr [[F64_ADDR]], align 8
+// CFINITEONLY-NEXT:    [[TMP3:%.*]] = load half, ptr [[F16_ADDR]], align 2
+// CFINITEONLY-NEXT:    [[TMP4:%.*]] = load <2 x float>, ptr [[V2F32_ADDR]], align 8
+// CFINITEONLY-NEXT:    [[TMP5:%.*]] = load <2 x double>, ptr [[V2F64_ADDR]], align 16
+// CFINITEONLY-NEXT:    [[TMP6:%.*]] = load <2 x half>, ptr [[V2F16_ADDR]], align 4
 // CFINITEONLY-NEXT:    [[CF32_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[CF32]], i32 0, i32 0
 // CFINITEONLY-NEXT:    [[CF32_REAL:%.*]] = load float, ptr [[CF32_REALP]], align 4
 // CFINITEONLY-NEXT:    [[CF32_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[CF32]], i32 0, i32 1
@@ -894,57 +869,59 @@ _Complex _Float16 defined_complex_func_f16_ret(_Complex _Float16 c) {
 // CFINITEONLY-NEXT:    [[CF64_IMAGP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[CF64]], i32 0, i32 1
 // CFINITEONLY-NEXT:    [[CF64_IMAG:%.*]] = load double, ptr [[CF64_IMAGP]], align 8
 // CFINITEONLY-NEXT:    [[CF16_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[CF16]], i32 0, i32 0
-// CFINITEONLY-NEXT:    [[CF16_REAL:%.*]] = load half, ptr [[CF16_REALP]], align 8
+// CFINITEONLY-NEXT:    [[CF16_REAL:%.*]] = load half, ptr [[CF16_REALP]], align 2
 // CFINITEONLY-NEXT:    [[CF16_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[CF16]], i32 0, i32 1
 // CFINITEONLY-NEXT:    [[CF16_IMAG:%.*]] = load half, ptr [[CF16_IMAGP]], align 2
-// CFINITEONLY-NEXT:    store <2 x float> [[TMP6]], ptr [[COERCE]], align 8
-// CFINITEONLY-NEXT:    [[TMP9:%.*]] = load double, ptr [[COERCE]], align 8
-// CFINITEONLY-NEXT:    store <2 x half> [[TMP8]], ptr [[COERCE3]], align 4
-// CFINITEONLY-NEXT:    [[TMP10:%.*]] = load i32, ptr [[COERCE3]], align 4
-// CFINITEONLY-NEXT:    [[COERCE4_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE4]], i32 0, i32 0
-// CFINITEONLY-NEXT:    [[COERCE4_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE4]], i32 0, i32 1
-// CFINITEONLY-NEXT:    store float [[CF32_REAL]], ptr [[COERCE4_REALP]], align 4
-// CFINITEONLY-NEXT:    store float [[CF32_IMAG]], ptr [[COERCE4_IMAGP]], align 4
-// CFINITEONLY-NEXT:    [[TMP11:%.*]] = load <2 x float>, ptr [[COERCE4]], align 4
+// CFINITEONLY-NEXT:    store <2 x float> [[TMP4]], ptr [[COERCE]], align 8
+// CFINITEONLY-NEXT:    [[TMP7:%.*]] = load double, ptr [[COERCE]], align 8
+// CFINITEONLY-NEXT:    [[COERCE2_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE2]], i32 0, i32 0
+// CFINITEONLY-NEXT:    [[COERCE2_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE2]], i32 0, i32 1
+// CFINITEONLY-NEXT:    store float [[CF32_REAL]], ptr [[COERCE2_REALP]], align 4
+// CFINITEONLY-NEXT:    store float [[CF32_IMAG]], ptr [[COERCE2_IMAGP]], align 4
+// CFINITEONLY-NEXT:    [[TMP8:%.*]] = load <2 x float>, ptr [[COERCE2]], align 4
 // CFINITEONLY-NEXT:    [[BYVAL_TEMP_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[BYVAL_TEMP]], i32 0, i32 0
 // CFINITEONLY-NEXT:    [[BYVAL_TEMP_IMAGP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[BYVAL_TEMP]], i32 0, i32 1
 // CFINITEONLY-NEXT:    store double [[CF64_REAL]], ptr [[BYVAL_TEMP_REALP]], align 8
 // CFINITEONLY-NEXT:    store double [[CF64_IMAG]], ptr [[BYVAL_TEMP_IMAGP]], align 8
-// CFINITEONLY-NEXT:    [[COERCE5_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[COERCE5]], i32 0, i32 0
-// CFINITEONLY-NEXT:    [[COERCE5_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[COERCE5]], i32 0, i32 1
-// CFINITEONLY-NEXT:    store half [[CF16_REAL]], ptr [[COERCE5_REALP]], align 2
-// CFINITEONLY-NEXT:    store half [[CF16_IMAG]], ptr [[COERCE5_IMAGP]], align 2
-// CFINITEONLY-NEXT:    [[TMP12:%.*]] = load <2 x half>, ptr [[COERCE5]], align 2
-// CFINITEONLY-NEXT:    [[CALL:%.*]] = call nnan ninf nofpclass(nan inf) float (float, ...) @variadic(float noundef nofpclass(nan inf) [[TMP2]], double noundef nofpclass(nan inf) [[CONV]], double noundef nofpclass(nan inf) [[TMP4]], half noundef nofpclass(nan inf) [[TMP5]], double noundef nofpclass(nan inf) [[TMP9]], <2 x double> noundef nofpclass(nan inf) [[TMP7]], i32 noundef [[TMP10]], <2 x float> noundef nofpclass(nan inf) [[TMP11]], ptr noundef byval({ double, double }) align 8 [[BYVAL_TEMP]], <2 x half> noundef nofpclass(nan inf) [[TMP12]])
+// CFINITEONLY-NEXT:    [[BYVAL_TEMP3_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[BYVAL_TEMP3]], i32 0, i32 0
+// CFINITEONLY-NEXT:    [[BYVAL_TEMP3_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[BYVAL_TEMP3]], i32 0, i32 1
+// CFINITEONLY-NEXT:    store half [[CF16_REAL]], ptr [[BYVAL_TEMP3_REALP]], align 8
+// CFINITEONLY-NEXT:    store half [[CF16_IMAG]], ptr [[BYVAL_TEMP3_IMAGP]], align 2
+// CFINITEONLY-NEXT:    [[CALL:%.*]] = call nnan ninf nofpclass(nan inf) float (float, ...) @variadic(float noundef nofpclass(nan inf) [[TMP0]], double noundef nofpclass(nan inf) [[CONV]], double noundef nofpclass(nan inf) [[TMP2]], half noundef nofpclass(nan inf) [[TMP3]], double noundef nofpclass(nan inf) [[TMP7]], <2 x double> noundef nofpclass(nan inf) [[TMP5]], <2 x half> noundef nofpclass(nan inf) [[TMP6]], <2 x float> noundef nofpclass(nan inf) [[TMP8]], ptr noundef byval({ double, double }) align 8 [[BYVAL_TEMP]], ptr noundef byval({ half, half }) align 8 [[BYVAL_TEMP3]])
 // CFINITEONLY-NEXT:    ret float [[CALL]]
 //
 // CLFINITEONLY: Function Attrs: convergent norecurse nounwind
 // CLFINITEONLY-LABEL: define dso_local nofpclass(nan inf) float @call_variadic
-// CLFINITEONLY-SAME: (float noundef nofpclass(nan inf) [[F32:%.*]], double noundef nofpclass(nan inf) [[F64:%.*]], half noundef nofpclass(nan inf) [[F16:%.*]], double noundef nofpclass(nan inf) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(nan inf) [[V2F64:%.*]], i32 noundef [[V2F16_COERCE:%.*]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE:%.*]], double noundef nofpclass(nan inf) [[CF64_COERCE0:%.*]], double noundef nofpclass(nan inf) [[CF64_COERCE1:%.*]], ptr nofree noundef readonly byval({ half, half }) align 8 captures(none) [[CF16:%.*]]) local_unnamed_addr #[[ATTR5]] {
+// CLFINITEONLY-SAME: (float noundef nofpclass(nan inf) [[F32:%.*]], double noundef nofpclass(nan inf) [[F64:%.*]], half noundef nofpclass(nan inf) [[F16:%.*]], double noundef nofpclass(nan inf) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(nan inf) [[V2F64:%.*]], <2 x half> noundef nofpclass(nan inf) [[V2F16:%.*]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE:%.*]], ptr nofree noundef readonly byval({ double, double }) align 8 captures(none) [[CF64:%.*]], <2 x half> noundef nofpclass(nan inf) [[CF16_COERCE:%.*]]) local_unnamed_addr #[[ATTR5]] {
 // CLFINITEONLY-NEXT:  entry:
 // CLFINITEONLY-NEXT:    [[BYVAL_TEMP:%.*]] = alloca { double, double }, align 8
+// CLFINITEONLY-NEXT:    [[BYVAL_TEMP3:%.*]] = alloca { half, half }, align 8
 // CLFINITEONLY-NEXT:    [[CONV:%.*]] = fpext nnan ninf float [[F32]] to double
-// CLFINITEONLY-NEXT:    [[CF16_REAL:%.*]] = load half, ptr [[CF16]], align 8
-// CLFINITEONLY-NEXT:    [[CF16_IMAGP:%.*]] = getelementptr inbounds nuw i8, ptr [[CF16]], i64 2
-// CLFINITEONLY-NEXT:    [[CF16_IMAG:%.*]] = load half, ptr [[CF16_IMAGP]], align 2
+// CLFINITEONLY-NEXT:    [[CF64_REAL:%.*]] = load double, ptr [[CF64]], align 8
+// CLFINITEONLY-NEXT:    [[CF64_IMAGP:%.*]] = getelementptr inbounds nuw i8, ptr [[CF64]], i64 8
+// CLFINITEONLY-NEXT:    [[CF64_IMAG:%.*]] = load double, ptr [[CF64_IMAGP]], align 8
+// CLFINITEONLY-NEXT:    [[CF16_SROA_0_0_VEC_EXTRACT:%.*]] = extractelement <2 x half> [[CF16_COERCE]], i64 0
+// CLFINITEONLY-NEXT:    [[CF16_SROA_0_2_VEC_EXTRACT:%.*]] = extractelement <2 x half> [[CF16_COERCE]], i64 1
 // CLFINITEONLY-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[BYVAL_TEMP]]) #[[ATTR12:[0-9]+]]
 // CLFINITEONLY-NEXT:    [[BYVAL_TEMP_IMAGP:%.*]] = getelementptr inbounds nuw i8, ptr [[BYVAL_TEMP]], i64 8
-// CLFINITEONLY-NEXT:    store double [[CF64_COERCE0]], ptr [[BYVAL_TEMP]], align 8
-// CLFINITEONLY-NEXT:    store double [[CF64_COERCE1]], ptr [[BYVAL_TEMP_IMAGP]], align 8
-// CLFINITEONLY-NEXT:    [[COERCE5_SROA_0_0_VEC_INSERT:%.*]] = insertelement <2 x half> poison, half [[CF16_REAL]], i64 0
-// CLFINITEONLY-NEXT:    [[COERCE5_SROA_0_2_VEC_INSERT:%.*]] = insertelement <2 x half> [[COERCE5_SROA_0_0_VEC_INSERT]], half [[CF16_IMAG]], i64 1
-// CLFINITEONLY-NEXT:    [[CALL:%.*]] = tail call nnan ninf nofpclass(nan inf) float (float, ...) @variadic(float noundef nofpclass(nan inf) [[F32]], double noundef nofpclass(nan inf) [[CONV]], double noundef nofpclass(nan inf) [[F64]], half noundef nofpclass(nan inf) [[F16]], double noundef nofpclass(nan inf) [[V2F32_COERCE]], <2 x double> noundef nofpclass(nan inf) [[V2F64]], i32 noundef [[V2F16_COERCE]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE]], ptr noundef nonnull byval({ double, double }) align 8 [[BYVAL_TEMP]], <2 x half> noundef nofpclass(nan inf) [[COERCE5_SROA_0_2_VEC_INSERT]]) #[[ATTR11]]
+// CLFINITEONLY-NEXT:    store double [[CF64_REAL]], ptr [[BYVAL_TEMP]], align 8
+// CLFINITEONLY-NEXT:    store double [[CF64_IMAG]], ptr [[BYVAL_TEMP_IMAGP]], align 8
+// CLFINITEONLY-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[BYVAL_TEMP3]]) #[[ATTR12]]
+// CLFINITEONLY-NEXT:    [[BYVAL_TEMP3_IMAGP:%.*]] = getelementptr inbounds nuw i8, ptr [[BYVAL_TEMP3]], i64 2
+// CLFINITEONLY-NEXT:    store half [[CF16_SROA_0_0_VEC_EXTRACT]], ptr [[BYVAL_TEMP3]], align 8
+// CLFINITEONLY-NEXT:    store half [[CF16_SROA_0_2_VEC_EXTRACT]], ptr [[BYVAL_TEMP3_IMAGP]], align 2
+// CLFINITEONLY-NEXT:    [[CALL:%.*]] = tail call nnan ninf nofpclass(nan inf) float (float, ...) @variadic(float noundef nofpclass(nan inf) [[F32]], double noundef nofpclass(nan inf) [[CONV]], double noundef nofpclass(nan inf) [[F64]], half noundef nofpclass(nan inf) [[F16]], double noundef nofpclass(nan inf) [[V2F32_COERCE]], <2 x double> noundef nofpclass(nan inf) [[V2F64]], <2 x half> noundef nofpclass(nan inf) [[V2F16]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE]], ptr noundef nonnull byval({ double, double }) align 8 [[BYVAL_TEMP]], ptr noundef nonnull byval({ half, half }) align 8 [[BYVAL_TEMP3]]) #[[ATTR11]]
 // CLFINITEONLY-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[BYVAL_TEMP]]) #[[ATTR12]]
+// CLFINITEONLY-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[BYVAL_TEMP3]]) #[[ATTR12]]
 // CLFINITEONLY-NEXT:    ret float [[CALL]]
 //
 // NONANS: Function Attrs: noinline nounwind optnone
 // NONANS-LABEL: define dso_local nofpclass(nan) float @call_variadic
-// NONANS-SAME: (float noundef nofpclass(nan) [[F32:%.*]], double noundef nofpclass(nan) [[F64:%.*]], half noundef nofpclass(nan) [[F16:%.*]], double noundef nofpclass(nan) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(nan) [[V2F64:%.*]], i32 noundef [[V2F16_COERCE:%.*]], <2 x float> noundef nofpclass(nan) [[CF32_COERCE:%.*]], double noundef nofpclass(nan) [[CF64_COERCE0:%.*]], double noundef nofpclass(nan) [[CF64_COERCE1:%.*]], ptr noundef byval({ half, half }) align 8 [[CF16:%.*]]) #[[ATTR2]] {
+// NONANS-SAME: (float noundef nofpclass(nan) [[F32:%.*]], double noundef nofpclass(nan) [[F64:%.*]], half noundef nofpclass(nan) [[F16:%.*]], double noundef nofpclass(nan) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(nan) [[V2F64:%.*]], <2 x half> noundef nofpclass(nan) [[V2F16:%.*]], <2 x float> noundef nofpclass(nan) [[CF32_COERCE:%.*]], ptr noundef byval({ double, double }) align 8 [[CF64:%.*]], <2 x half> noundef nofpclass(nan) [[CF16_COERCE:%.*]]) #[[ATTR2]] {
 // NONANS-NEXT:  entry:
 // NONANS-NEXT:    [[V2F32:%.*]] = alloca <2 x float>, align 8
-// NONANS-NEXT:    [[V2F16:%.*]] = alloca <2 x half>, align 4
 // NONANS-NEXT:    [[CF32:%.*]] = alloca { float, float }, align 4
-// NONANS-NEXT:    [[CF64:%.*]] = alloca { double, double }, align 8
+// NONANS-NEXT:    [[CF16:%.*]] = alloca { half, half }, align 2
 // NONANS-NEXT:    [[F32_ADDR:%.*]] = alloca float, align 4
 // NONANS-NEXT:    [[F64_ADDR:%.*]] = alloca double, align 8
 // NONANS-NEXT:    [[F16_ADDR:%.*]] = alloca half, align 2
@@ -952,33 +929,27 @@ _Complex _Float16 defined_complex_func_f16_ret(_Complex _Float16 c) {
 // NONANS-NEXT:    [[V2F64_ADDR:%.*]] = alloca <2 x double>, align 16
 // NONANS-NEXT:    [[V2F16_ADDR:%.*]] = alloca <2 x half>, align 4
 // NONANS-NEXT:    [[COERCE:%.*]] = alloca <2 x float>, align 8
-// NONANS-NEXT:    [[COERCE3:%.*]] = alloca <2 x half>, align 4
-// NONANS-NEXT:    [[COERCE4:%.*]] = alloca { float, float }, align 4
+// NONANS-NEXT:    [[COERCE2:%.*]] = alloca { float, float }, align 4
 // NONANS-NEXT:    [[BYVAL_TEMP:%.*]] = alloca { double, double }, align 8
-// NONANS-NEXT:    [[COERCE5:%.*]] = alloca { half, half }, align 2
+// NONANS-NEXT:    [[BYVAL_TEMP3:%.*]] = alloca { half, half }, align 8
 // NONANS-NEXT:    store double [[V2F32_COERCE]], ptr [[V2F32]], align 8
 // NONANS-NEXT:    [[V2F321:%.*]] = load <2 x float>, ptr [[V2F32]], align 8
-// NONANS-NEXT:    store i32 [[V2F16_COERCE]], ptr [[V2F16]], align 4
-// NONANS-NEXT:    [[V2F162:%.*]] = load <2 x half>, ptr [[V2F16]], align 4
 // NONANS-NEXT:    store <2 x float> [[CF32_COERCE]], ptr [[CF32]], align 4
-// NONANS-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[CF64]], i32 0, i32 0
-// NONANS-NEXT:    store double [[CF64_COERCE0]], ptr [[TMP0]], align 8
-// NONANS-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[CF64]], i32 0, i32 1
-// NONANS-NEXT:    store double [[CF64_COERCE1]], ptr [[TMP1]], align 8
+// NONANS-NEXT:    store <2 x half> [[CF16_COERCE]], ptr [[CF16]], align 2
 // NONANS-NEXT:    store float [[F32]], ptr [[F32_ADDR]], align 4
 // NONANS-NEXT:    store double [[F64]], ptr [[F64_ADDR]], align 8
 // NONANS-NEXT:    store half [[F16]], ptr [[F16_ADDR]], align 2
 // NONANS-NEXT:    store <2 x float> [[V2F321]], ptr [[V2F32_ADDR]], align 8
 // NONANS-NEXT:    store <2 x double> [[V2F64]], ptr [[V2F64_ADDR]], align 16
-// NONANS-NEXT:    store <2 x half> [[V2F162]], ptr [[V2F16_ADDR]], align 4
-// NONANS-NEXT:    [[TMP2:%.*]] = load float, ptr [[F32_ADDR]], align 4
-// NONANS-NEXT:    [[TMP3:%.*]] = load float, ptr [[F32_ADDR]], align 4
-// NONANS-NEXT:    [[CONV:%.*]] = fpext nnan float [[TMP3]] to double
-// NONANS-NEXT:    [[TMP4:%.*]] = load double, ptr [[F64_ADDR]], align 8
-// NONANS-NEXT:    [[TMP5:%.*]] = load half, ptr [[F16_ADDR]], align 2
-// NONANS-NEXT:    [[TMP6:%.*]] = load <2 x float>, ptr [[V2F32_ADDR]], align 8
-// NONANS-NEXT:    [[TMP7:%.*]] = load <2 x double>, ptr [[V2F64_ADDR]], align 16
-// NONANS-NEXT:    [[TMP8:%.*]] = load <2 x half>, ptr [[V2F16_ADDR]], align 4
+// NONANS-NEXT:    store <2 x half> [[V2F16]], ptr [[V2F16_ADDR]], align 4
+// NONANS-NEXT:    [[TMP0:%.*]] = load float, ptr [[F32_ADDR]], align 4
+// NONANS-NEXT:    [[TMP1:%.*]] = load float, ptr [[F32_ADDR]], align 4
+// NONANS-NEXT:    [[CONV:%.*]] = fpext nnan float [[TMP1]] to double
+// NONANS-NEXT:    [[TMP2:%.*]] = load double, ptr [[F64_ADDR]], align 8
+// NONANS-NEXT:    [[TMP3:%.*]] = load half, ptr [[F16_ADDR]], align 2
+// NONANS-NEXT:    [[TMP4:%.*]] = load <2 x float>, ptr [[V2F32_ADDR]], align 8
+// NONANS-NEXT:    [[TMP5:%.*]] = load <2 x double>, ptr [[V2F64_ADDR]], align 16
+// NONANS-NEXT:    [[TMP6:%.*]] = load <2 x half>, ptr [[V2F16_ADDR]], align 4
 // NONANS-NEXT:    [[CF32_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[CF32]], i32 0, i32 0
 // NONANS-NEXT:    [[CF32_REAL:%.*]] = load float, ptr [[CF32_REALP]], align 4
 // NONANS-NEXT:    [[CF32_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[CF32]], i32 0, i32 1
@@ -988,38 +959,34 @@ _Complex _Float16 defined_complex_func_f16_ret(_Complex _Float16 c) {
 // NONANS-NEXT:    [[CF64_IMAGP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[CF64]], i32 0, i32 1
 // NONANS-NEXT:    [[CF64_IMAG:%.*]] = load double, ptr [[CF64_IMAGP]], align 8
 // NONANS-NEXT:    [[CF16_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[CF16]], i32 0, i32 0
-// NONANS-NEXT:    [[CF16_REAL:%.*]] = load half, ptr [[CF16_REALP]], align 8
+// NONANS-NEXT:    [[CF16_REAL:%.*]] = load half, ptr [[CF16_REALP]], align 2
 // NONANS-NEXT:    [[CF16_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[CF16]], i32 0, i32 1
 // NONANS-NEXT:    [[CF16_IMAG:%.*]] = load half, ptr [[CF16_IMAGP]], align 2
-// NONANS-NEXT:    store <2 x float> [[TMP6]], ptr [[COERCE]], align 8
-// NONANS-NEXT:    [[TMP9:%.*]] = load double, ptr [[COERCE]], align 8
-// NONANS-NEXT:    store <2 x half> [[TMP8]], ptr [[COERCE3]], align 4
-// NONANS-NEXT:    [[TMP10:%.*]] = load i32, ptr [[COERCE3]], align 4
-// NONANS-NEXT:    [[COERCE4_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE4]], i32 0, i32 0
-// NONANS-NEXT:    [[COERCE4_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE4]], i32 0, i32 1
-// NONANS-NEXT:    store float [[CF32_REAL]], ptr [[COERCE4_REALP]], align 4
-// NONANS-NEXT:    store float [[CF32_IMAG]], ptr [[COERCE4_IMAGP]], align 4
-// NONANS-NEXT:    [[TMP11:%.*]] = load <2 x float>, ptr [[COERCE4]], align 4
+// NONANS-NEXT:    store <2 x float> [[TMP4]], ptr [[COERCE]], align 8
+// NONANS-NEXT:    [[TMP7:%.*]] = load double, ptr [[COERCE]], align 8
+// NONANS-NEXT:    [[COERCE2_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE2]], i32 0, i32 0
+// NONANS-NEXT:    [[COERCE2_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE2]], i32 0, i32 1
+// NONANS-NEXT:    store float [[CF32_REAL]], ptr [[COERCE2_REALP]], align 4
+// NONANS-NEXT:    store float [[CF32_IMAG]], ptr [[COERCE2_IMAGP]], align 4
+// NONANS-NEXT:    [[TMP8:%.*]] = load <2 x float>, ptr [[COERCE2]], align 4
 // NONANS-NEXT:    [[BYVAL_TEMP_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[BYVAL_TEMP]], i32 0, i32 0
 // NONANS-NEXT:    [[BYVAL_TEMP_IMAGP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[BYVAL_TEMP]], i32 0, i32 1
 // NONANS-NEXT:    store double [[CF64_REAL]], ptr [[BYVAL_TEMP_REALP]], align 8
 // NONANS-NEXT:    store double [[CF64_IMAG]], ptr [[BYVAL_TEMP_IMAGP]], align 8
-// NONANS-NEXT:    [[COERCE5_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[COERCE5]], i32 0, i32 0
-// NONANS-NEXT:    [[COERCE5_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[COERCE5]], i32 0, i32 1
-// NONANS-NEXT:    store half [[CF16_REAL]], ptr [[COERCE5_REALP]], align 2
-// NONANS-NEXT:    store half [[CF16_IMAG]], ptr [[COERCE5_IMAGP]], align 2
-// NONANS-NEXT:    [[TMP12:%.*]] = load <2 x half>, ptr [[COERCE5]], align 2
-// NONANS-NEXT:    [[CALL:%.*]] = call nnan nofpclass(nan) float (float, ...) @variadic(float noundef nofpclass(nan) [[TMP2]], double noundef nofpclass(nan) [[CONV]], double noundef nofpclass(nan) [[TMP4]], half noundef nofpclass(nan) [[TMP5]], double noundef nofpclass(nan) [[TMP9]], <2 x double> noundef nofpclass(nan) [[TMP7]], i32 noundef [[TMP10]], <2 x float> noundef nofpclass(nan) [[TMP11]], ptr noundef byval({ double, double }) align 8 [[BYVAL_TEMP]], <2 x half> noundef nofpclass(nan) [[TMP12]])
+// NONANS-NEXT:    [[BYVAL_TEMP3_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[BYVAL_TEMP3]], i32 0, i32 0
+// NONANS-NEXT:    [[BYVAL_TEMP3_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[BYVAL_TEMP3]], i32 0, i32 1
+// NONANS-NEXT:    store half [[CF16_REAL]], ptr [[BYVAL_TEMP3_REALP]], align 8
+// NONANS-NEXT:    store half [[CF16_IMAG]], ptr [[BYVAL_TEMP3_IMAGP]], align 2
+// NONANS-NEXT:    [[CALL:%.*]] = call nnan nofpclass(nan) float (float, ...) @variadic(float noundef nofpclass(nan) [[TMP0]], double noundef nofpclass(nan) [[CONV]], double noundef nofpclass(nan) [[TMP2]], half noundef nofpclass(nan) [[TMP3]], double noundef nofpclass(nan) [[TMP7]], <2 x double> noundef nofpclass(nan) [[TMP5]], <2 x half> noundef nofpclass(nan) [[TMP6]], <2 x float> noundef nofpclass(nan) [[TMP8]], ptr noundef byval({ double, double }) align 8 [[BYVAL_TEMP]], ptr noundef byval({ half, half }) align 8 [[BYVAL_TEMP3]])
 // NONANS-NEXT:    ret float [[CALL]]
 //
 // NOINFS: Function Attrs: noinline nounwind optnone
 // NOINFS-LABEL: define dso_local nofpclass(inf) float @call_variadic
-// NOINFS-SAME: (float noundef nofpclass(inf) [[F32:%.*]], double noundef nofpclass(inf) [[F64:%.*]], half noundef nofpclass(inf) [[F16:%.*]], double noundef nofpclass(inf) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(inf) [[V2F64:%.*]], i32 noundef [[V2F16_COERCE:%.*]], <2 x float> noundef nofpclass(inf) [[CF32_COERCE:%.*]], double noundef nofpclass(inf) [[CF64_COERCE0:%.*]], double noundef nofpclass(inf) [[CF64_COERCE1:%.*]], ptr noundef byval({ half, half }) align 8 [[CF16:%.*]]) #[[ATTR2]] {
+// NOINFS-SAME: (float noundef nofpclass(inf) [[F32:%.*]], double noundef nofpclass(inf) [[F64:%.*]], half noundef nofpclass(inf) [[F16:%.*]], double noundef nofpclass(inf) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(inf) [[V2F64:%.*]], <2 x half> noundef nofpclass(inf) [[V2F16:%.*]], <2 x float> noundef nofpclass(inf) [[CF32_COERCE:%.*]], ptr noundef byval({ double, double }) align 8 [[CF64:%.*]], <2 x half> noundef nofpclass(inf) [[CF16_COERCE:%.*]]) #[[ATTR2]] {
 // NOINFS-NEXT:  entry:
 // NOINFS-NEXT:    [[V2F32:%.*]] = alloca <2 x float>, align 8
-// NOINFS-NEXT:    [[V2F16:%.*]] = alloca <2 x half>, align 4
 // NOINFS-NEXT:    [[CF32:%.*]] = alloca { float, float }, align 4
-// NOINFS-NEXT:    [[CF64:%.*]] = alloca { double, double }, align 8
+// NOINFS-NEXT:    [[CF16:%.*]] = alloca { half, half }, align 2
 // NOINFS-NEXT:    [[F32_ADDR:%.*]] = alloca float, align 4
 // NOINFS-NEXT:    [[F64_ADDR:%.*]] = alloca double, align 8
 // NOINFS-NEXT:    [[F16_ADDR:%.*]] = alloca half, align 2
@@ -1027,33 +994,27 @@ _Complex _Float16 defined_complex_func_f16_ret(_Complex _Float16 c) {
 // NOINFS-NEXT:    [[V2F64_ADDR:%.*]] = alloca <2 x double>, align 16
 // NOINFS-NEXT:    [[V2F16_ADDR:%.*]] = alloca <2 x half>, align 4
 // NOINFS-NEXT:    [[COERCE:%.*]] = alloca <2 x float>, align 8
-// NOINFS-NEXT:    [[COERCE3:%.*]] = alloca <2 x half>, align 4
-// NOINFS-NEXT:    [[COERCE4:%.*]] = alloca { float, float }, align 4
+// NOINFS-NEXT:    [[COERCE2:%.*]] = alloca { float, float }, align 4
 // NOINFS-NEXT:    [[BYVAL_TEMP:%.*]] = alloca { double, double }, align 8
-// NOINFS-NEXT:    [[COERCE5:%.*]] = alloca { half, half }, align 2
+// NOINFS-NEXT:    [[BYVAL_TEMP3:%.*]] = alloca { half, half }, align 8
 // NOINFS-NEXT:    store double [[V2F32_COERCE]], ptr [[V2F32]], align 8
 // NOINFS-NEXT:    [[V2F321:%.*]] = load <2 x float>, ptr [[V2F32]], align 8
-// NOINFS-NEXT:    store i32 [[V2F16_COERCE]], ptr [[V2F16]], align 4
-// NOINFS-NEXT:    [[V2F162:%.*]] = load <2 x half>, ptr [[V2F16]], align 4
 // NOINFS-NEXT:    store <2 x float> [[CF32_COERCE]], ptr [[CF32]], align 4
-// NOINFS-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[CF64]], i32 0, i32 0
-// NOINFS-NEXT:    store double [[CF64_COERCE0]], ptr [[TMP0]], align 8
-// NOINFS-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[CF64]], i32 0, i32 1
-// NOINFS-NEXT:    store double [[CF64_COERCE1]], ptr [[TMP1]], align 8
+// NOINFS-NEXT:    store <2 x half> [[CF16_COERCE]], ptr [[CF16]], align 2
 // NOINFS-NEXT:    store float [[F32]], ptr [[F32_ADDR]], align 4
 // NOINFS-NEXT:    store double [[F64]], ptr [[F64_ADDR]], align 8
 // NOINFS-NEXT:    store half [[F16]], ptr [[F16_ADDR]], align 2
 // NOINFS-NEXT:    store <2 x float> [[V2F321]], ptr [[V2F32_ADDR]], align 8
 // NOINFS-NEXT:    store <2 x double> [[V2F64]], ptr [[V2F64_ADDR]], align 16
-// NOINFS-NEXT:    store <2 x half> [[V2F162]], ptr [[V2F16_ADDR]], align 4
-// NOINFS-NEXT:    [[TMP2:%.*]] = load float, ptr [[F32_ADDR]], align 4
-// NOINFS-NEXT:    [[TMP3:%.*]] = load float, ptr [[F32_ADDR]], align 4
-// NOINFS-NEXT:    [[CONV:%.*]] = fpext ninf float [[TMP3]] to double
-// NOINFS-NEXT:    [[TMP4:%.*]] = load double, ptr [[F64_ADDR]], align 8
-// NOINFS-NEXT:    [[TMP5:%.*]] = load half, ptr [[F16_ADDR]], align 2
-// NOINFS-NEXT:    [[TMP6:%.*]] = load <2 x float>, ptr [[V2F32_ADDR]], align 8
-// NOINFS-NEXT:    [[TMP7:%.*]] = load <2 x double>, ptr [[V2F64_ADDR]], align 16
-// NOINFS-NEXT:    [[TMP8:%.*]] = load <2 x half>, ptr [[V2F16_ADDR]], align 4
+// NOINFS-NEXT:    store <2 x half> [[V2F16]], ptr [[V2F16_ADDR]], align 4
+// NOINFS-NEXT:    [[TMP0:%.*]] = load float, ptr [[F32_ADDR]], align 4
+// NOINFS-NEXT:    [[TMP1:%.*]] = load float, ptr [[F32_ADDR]], align 4
+// NOINFS-NEXT:    [[CONV:%.*]] = fpext ninf float [[TMP1]] to double
+// NOINFS-NEXT:    [[TMP2:%.*]] = load double, ptr [[F64_ADDR]], align 8
+// NOINFS-NEXT:    [[TMP3:%.*]] = load half, ptr [[F16_ADDR]], align 2
+// NOINFS-NEXT:    [[TMP4:%.*]] = load <2 x float>, ptr [[V2F32_ADDR]], align 8
+// NOINFS-NEXT:    [[TMP5:%.*]] = load <2 x double>, ptr [[V2F64_ADDR]], align 16
+// NOINFS-NEXT:    [[TMP6:%.*]] = load <2 x half>, ptr [[V2F16_ADDR]], align 4
 // NOINFS-NEXT:    [[CF32_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[CF32]], i32 0, i32 0
 // NOINFS-NEXT:    [[CF32_REAL:%.*]] = load float, ptr [[CF32_REALP]], align 4
 // NOINFS-NEXT:    [[CF32_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[CF32]], i32 0, i32 1
@@ -1063,28 +1024,25 @@ _Complex _Float16 defined_complex_func_f16_ret(_Complex _Float16 c) {
 // NOINFS-NEXT:    [[CF64_IMAGP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[CF64]], i32 0, i32 1
 // NOINFS-NEXT:    [[CF64_IMAG:%.*]] = load double, ptr [[CF64_IMAGP]], align 8
 // NOINFS-NEXT:    [[CF16_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[CF16]], i32 0, i32 0
-// NOINFS-NEXT:    [[CF16_REAL:%.*]] = load half, ptr [[CF16_REALP]], align 8
+// NOINFS-NEXT:    [[CF16_REAL:%.*]] = load half, ptr [[CF16_REALP]], align 2
 // NOINFS-NEXT:    [[CF16_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[CF16]], i32 0, i32 1
 // NOINFS-NEXT:    [[CF16_IMAG:%.*]] = load half, ptr [[CF16_IMAGP]], align 2
-// NOINFS-NEXT:    store <2 x float> [[TMP6]], ptr [[COERCE]], align 8
-// NOINFS-NEXT:    [[TMP9:%.*]] = load double, ptr [[COERCE]], align 8
-// NOINFS-NEXT:    store <2 x half> [[TMP8]], ptr [[COERCE3]], align 4
-// NOINFS-NEXT:    [[TMP10:%.*]] = load i32, ptr [[COERCE3]], align 4
-// NOINFS-NEXT:    [[COERCE4_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE4]], i32 0, i32 0
-// NOINFS-NEXT:    [[COERCE4_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE4]], i32 0, i32 1
-// NOINFS-NEXT:    store float [[CF32_REAL]], ptr [[COERCE4_REALP]], align 4
-// NOINFS-NEXT:    store float [[CF32_IMAG]], ptr [[COERCE4_IMAGP]], align 4
-// NOINFS-NEXT:    [[TMP11:%.*]] = load <2 x float>, ptr [[COERCE4]], align 4
+// NOINFS-NEXT:    store <2 x float> [[TMP4]], ptr [[COERCE]], align 8
+// NOINFS-NEXT:    [[TMP7:%.*]] = load double, ptr [[COERCE]], align 8
+// NOINFS-NEXT:    [[COERCE2_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE2]], i32 0, i32 0
+// NOINFS-NEXT:    [[COERCE2_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE2]], i32 0, i32 1
+// NOINFS-NEXT:    store float [[CF32_REAL]], ptr [[COERCE2_REALP]], align 4
+// NOINFS-NEXT:    store float [[CF32_IMAG]], ptr [[COERCE2_IMAGP]], align 4
+// NOINFS-NEXT:    [[TMP8:%.*]] = load <2 x float>, ptr [[COERCE2]], align 4
 // NOINFS-NEXT:    [[BYVAL_TEMP_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[BYVAL_TEMP]], i32 0, i32 0
 // NOINFS-NEXT:    [[BYVAL_TEMP_IMAGP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[BYVAL_TEMP]], i32 0, i32 1
 // NOINFS-NEXT:    store double [[CF64_REAL]], ptr [[BYVAL_TEMP_REALP]], align 8
 // NOINFS-NEXT:    store double [[CF64_IMAG]], ptr [[BYVAL_TEMP_IMAGP]], align 8
-// NOINFS-NEXT:    [[COERCE5_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[COERCE5]], i32 0, i32 0
-// NOINFS-NEXT:    [[COERCE5_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[COERCE5]], i32 0, i32 1
-// NOINFS-NEXT:    store half [[CF16_REAL]], ptr [[COERCE5_REALP]], align 2
-// NOINFS-NEXT:    store half [[CF16_IMAG]], ptr [[COERCE5_IMAGP]], align 2
-// NOINFS-NEXT:    [[TMP12:%.*]] = load <2 x half>, ptr [[COERCE5]], align 2
-// NOINFS-NEXT:    [[CALL:%.*]] = call ninf nofpclass(inf) float (float, ...) @variadic(float noundef nofpclass(inf) [[TMP2]], double noundef nofpclass(inf) [[CONV]], double noundef nofpclass(inf) [[TMP4]], half noundef nofpclass(inf) [[TMP5]], double noundef nofpclass(inf) [[TMP9]], <2 x double> noundef nofpclass(inf) [[TMP7]], i32 noundef [[TMP10]], <2 x float> noundef nofpclass(inf) [[TMP11]], ptr noundef byval({ double, double }) align 8 [[BYVAL_TEMP]], <2 x half> noundef nofpclass(inf) [[TMP12]])
+// NOINFS-NEXT:    [[BYVAL_TEMP3_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[BYVAL_TEMP3]], i32 0, i32 0
+// NOINFS-NEXT:    [[BYVAL_TEMP3_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[BYVAL_TEMP3]], i32 0, i32 1
+// NOINFS-NEXT:    store half [[CF16_REAL]], ptr [[BYVAL_TEMP3_REALP]], align 8
+// NOINFS-NEXT:    store half [[CF16_IMAG]], ptr [[BYVAL_TEMP3_IMAGP]], align 2
+// NOINFS-NEXT:    [[CALL:%.*]] = call ninf nofpclass(inf) float (float, ...) @variadic(float noundef nofpclass(inf) [[TMP0]], double noundef nofpclass(inf) [[CONV]], double noundef nofpclass(inf) [[TMP2]], half noundef nofpclass(inf) [[TMP3]], double noundef nofpclass(inf) [[TMP7]], <2 x double> noundef nofpclass(inf) [[TMP5]], <2 x half> noundef nofpclass(inf) [[TMP6]], <2 x float> noundef nofpclass(inf) [[TMP8]], ptr noundef byval({ double, double }) align 8 [[BYVAL_TEMP]], ptr noundef byval({ half, half }) align 8 [[BYVAL_TEMP3]])
 // NOINFS-NEXT:    ret float [[CALL]]
 //
 float call_variadic(float f32, double f64, _Float16 f16,
@@ -1095,12 +1053,11 @@ float call_variadic(float f32, double f64, _Float16 f16,
 
 // CFINITEONLY: Function Attrs: noinline nounwind optnone
 // CFINITEONLY-LABEL: define dso_local nofpclass(nan inf) float @call_variadic_indirect
-// CFINITEONLY-SAME: (ptr noundef [[FPTR:%.*]], float noundef nofpclass(nan inf) [[F32:%.*]], double noundef nofpclass(nan inf) [[F64:%.*]], half noundef nofpclass(nan inf) [[F16:%.*]], double noundef nofpclass(nan inf) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(nan inf) [[V2F64:%.*]], i32 noundef [[V2F16_COERCE:%.*]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE:%.*]], double noundef nofpclass(nan inf) [[CF64_COERCE0:%.*]], double noundef nofpclass(nan inf) [[CF64_COERCE1:%.*]], ptr noundef byval({ half, half }) align 8 [[CF16:%.*]]) #[[ATTR2]] {
+// CFINITEONLY-SAME: (ptr noundef [[FPTR:%.*]], float noundef nofpclass(nan inf) [[F32:%.*]], double noundef nofpclass(nan inf) [[F64:%.*]], half noundef nofpclass(nan inf) [[F16:%.*]], double noundef nofpclass(nan inf) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(nan inf) [[V2F64:%.*]], <2 x half> noundef nofpclass(nan inf) [[V2F16:%.*]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE:%.*]], ptr noundef byval({ double, double }) align 8 [[CF64:%.*]], <2 x half> noundef nofpclass(nan inf) [[CF16_COERCE:%.*]]) #[[ATTR2]] {
 // CFINITEONLY-NEXT:  entry:
 // CFINITEONLY-NEXT:    [[V2F32:%.*]] = alloca <2 x float>, align 8
-// CFINITEONLY-NEXT:    [[V2F16:%.*]] = alloca <2 x half>, align 4
 // CFINITEONLY-NEXT:    [[CF32:%.*]] = alloca { float, float }, align 4
-// CFINITEONLY-NEXT:    [[CF64:%.*]] = alloca { double, double }, align 8
+// CFINITEONLY-NEXT:    [[CF16:%.*]] = alloca { half, half }, align 2
 // CFINITEONLY-NEXT:    [[FPTR_ADDR:%.*]] = alloca ptr, align 8
 // CFINITEONLY-NEXT:    [[F32_ADDR:%.*]] = alloca float, align 4
 // CFINITEONLY-NEXT:    [[F64_ADDR:%.*]] = alloca double, align 8
@@ -1109,35 +1066,29 @@ float call_variadic(float f32, double f64, _Float16 f16,
 // CFINITEONLY-NEXT:    [[V2F64_ADDR:%.*]] = alloca <2 x double>, align 16
 // CFINITEONLY-NEXT:    [[V2F16_ADDR:%.*]] = alloca <2 x half>, align 4
 // CFINITEONLY-NEXT:    [[COERCE:%.*]] = alloca <2 x float>, align 8
-// CFINITEONLY-NEXT:    [[COERCE3:%.*]] = alloca <2 x half>, align 4
-// CFINITEONLY-NEXT:    [[COERCE4:%.*]] = alloca { float, float }, align 4
+// CFINITEONLY-NEXT:    [[COERCE2:%.*]] = alloca { float, float }, align 4
 // CFINITEONLY-NEXT:    [[BYVAL_TEMP:%.*]] = alloca { double, double }, align 8
-// CFINITEONLY-NEXT:    [[COERCE5:%.*]] = alloca { half, half }, align 2
+// CFINITEONLY-NEXT:    [[BYVAL_TEMP3:%.*]] = alloca { half, half }, align 8
 // CFINITEONLY-NEXT:    store double [[V2F32_COERCE]], ptr [[V2F32]], align 8
 // CFINITEONLY-NEXT:    [[V2F321:%.*]] = load <2 x float>, ptr [[V2F32]], align 8
-// CFINITEONLY-NEXT:    store i32 [[V2F16_COERCE]], ptr [[V2F16]], align 4
-// CFINITEONLY-NEXT:    [[V2F162:%.*]] = load <2 x half>, ptr [[V2F16]], align 4
 // CFINITEONLY-NEXT:    store <2 x float> [[CF32_COERCE]], ptr [[CF32]], align 4
-// CFINITEONLY-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[CF64]], i32 0, i32 0
-// CFINITEONLY-NEXT:    store double [[CF64_COERCE0]], ptr [[TMP0]], align 8
-// CFINITEONLY-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[CF64]], i32 0, i32 1
-// CFINITEONLY-NEXT:    store double [[CF64_COERCE1]], ptr [[TMP1]], align 8
+// CFINITEONLY-NEXT:    store <2 x half> [[CF16_COERCE]], ptr [[CF16]], align 2
 // CFINITEONLY-NEXT:    store ptr [[FPTR]], ptr [[FPTR_ADDR]], align 8
 // CFINITEONLY-NEXT:    store float [[F32]], ptr [[F32_ADDR]], align 4
 // CFINITEONLY-NEXT:    store double [[F64]], ptr [[F64_ADDR]], align 8
 // CFINITEONLY-NEXT:    store half [[F16]], ptr [[F16_ADDR]], align 2
 // CFINITEONLY-NEXT:    store <2 x float> [[V2F321]], ptr [[V2F32_ADDR]], align 8
 // CFINITEONLY-NEXT:    store <2 x double> [[V2F64]], ptr [[V2F64_ADDR]], align 16
-// CFINITEONLY-NEXT:    store <2 x half> [[V2F162]], ptr [[V2F16_ADDR]], align 4
-// CFINITEONLY-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[FPTR_ADDR]], align 8
-// CFINITEONLY-NEXT:    [[TMP3:%.*]] = load float, ptr [[F32_ADDR]], align 4
-// CFINITEONLY-NEXT:    [[TMP4:%.*]] = load float, ptr [[F32_ADDR]], align 4
-// CFINITEONLY-NEXT:    [[CONV:%.*]] = fpext nnan ninf float [[TMP4]] to double
-// CFINITEONLY-NEXT:    [[TMP5:%.*]] = load double, ptr [[F64_ADDR]], align 8
-// CFINITEONLY-NEXT:    [[TMP6:%.*]] = load half, ptr [[F16_ADDR]], align 2
-// CFINITEONLY-NEXT:    [[TMP7:%.*]] = load <2 x float>, ptr [[V2F32_ADDR]], align 8
-// CFINITEONLY-NEXT:    [[TMP8:%.*]] = load <2 x double>, ptr [[V2F64_ADDR]], align 16
-// CFINITEONLY-NEXT:    [[TMP9:%.*]] = load <2 x half>, ptr [[V2F16_ADDR]], align 4
+// CFINITEONLY-NEXT:    store <2 x half> [[V2F16]], ptr [[V2F16_ADDR]], align 4
+// CFINITEONLY-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[FPTR_ADDR]], align 8
+// CFINITEONLY-NEXT:    [[TMP1:%.*]] = load float, ptr [[F32_ADDR]], align 4
+// CFINITEONLY-NEXT:    [[TMP2:%.*]] = load float, ptr [[F32_ADDR]], align 4
+// CFINITEONLY-NEXT:    [[CONV:%.*]] = fpext nnan ninf float [[TMP2]] to double
+// CFINITEONLY-NEXT:    [[TMP3:%.*]] = load double, ptr [[F64_ADDR]], align 8
+// CFINITEONLY-NEXT:    [[TMP4:%.*]] = load half, ptr [[F16_ADDR]], align 2
+// CFINITEONLY-NEXT:    [[TMP5:%.*]] = load <2 x float>, ptr [[V2F32_ADDR]], align 8
+// CFINITEONLY-NEXT:    [[TMP6:%.*]] = load <2 x double>, ptr [[V2F64_ADDR]], align 16
+// CFINITEONLY-NEXT:    [[TMP7:%.*]] = load <2 x half>, ptr [[V2F16_ADDR]], align 4
 // CFINITEONLY-NEXT:    [[CF32_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[CF32]], i32 0, i32 0
 // CFINITEONLY-NEXT:    [[CF32_REAL:%.*]] = load float, ptr [[CF32_REALP]], align 4
 // CFINITEONLY-NEXT:    [[CF32_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[CF32]], i32 0, i32 1
@@ -1147,57 +1098,59 @@ float call_variadic(float f32, double f64, _Float16 f16,
 // CFINITEONLY-NEXT:    [[CF64_IMAGP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[CF64]], i32 0, i32 1
 // CFINITEONLY-NEXT:    [[CF64_IMAG:%.*]] = load double, ptr [[CF64_IMAGP]], align 8
 // CFINITEONLY-NEXT:    [[CF16_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[CF16]], i32 0, i32 0
-// CFINITEONLY-NEXT:    [[CF16_REAL:%.*]] = load half, ptr [[CF16_REALP]], align 8
+// CFINITEONLY-NEXT:    [[CF16_REAL:%.*]] = load half, ptr [[CF16_REALP]], align 2
 // CFINITEONLY-NEXT:    [[CF16_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[CF16]], i32 0, i32 1
 // CFINITEONLY-NEXT:    [[CF16_IMAG:%.*]] = load half, ptr [[CF16_IMAGP]], align 2
-// CFINITEONLY-NEXT:    store <2 x float> [[TMP7]], ptr [[COERCE]], align 8
-// CFINITEONLY-NEXT:    [[TMP10:%.*]] = load double, ptr [[COERCE]], align 8
-// CFINITEONLY-NEXT:    store <2 x half> [[TMP9]], ptr [[COERCE3]], align 4
-// CFINITEONLY-NEXT:    [[TMP11:%.*]] = load i32, ptr [[COERCE3]], align 4
-// CFINITEONLY-NEXT:    [[COERCE4_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE4]], i32 0, i32 0
-// CFINITEONLY-NEXT:    [[COERCE4_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE4]], i32 0, i32 1
-// CFINITEONLY-NEXT:    store float [[CF32_REAL]], ptr [[COERCE4_REALP]], align 4
-// CFINITEONLY-NEXT:    store float [[CF32_IMAG]], ptr [[COERCE4_IMAGP]], align 4
-// CFINITEONLY-NEXT:    [[TMP12:%.*]] = load <2 x float>, ptr [[COERCE4]], align 4
+// CFINITEONLY-NEXT:    store <2 x float> [[TMP5]], ptr [[COERCE]], align 8
+// CFINITEONLY-NEXT:    [[TMP8:%.*]] = load double, ptr [[COERCE]], align 8
+// CFINITEONLY-NEXT:    [[COERCE2_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE2]], i32 0, i32 0
+// CFINITEONLY-NEXT:    [[COERCE2_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE2]], i32 0, i32 1
+// CFINITEONLY-NEXT:    store float [[CF32_REAL]], ptr [[COERCE2_REALP]], align 4
+// CFINITEONLY-NEXT:    store float [[CF32_IMAG]], ptr [[COERCE2_IMAGP]], align 4
+// CFINITEONLY-NEXT:    [[TMP9:%.*]] = load <2 x float>, ptr [[COERCE2]], align 4
 // CFINITEONLY-NEXT:    [[BYVAL_TEMP_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[BYVAL_TEMP]], i32 0, i32 0
 // CFINITEONLY-NEXT:    [[BYVAL_TEMP_IMAGP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[BYVAL_TEMP]], i32 0, i32 1
 // CFINITEONLY-NEXT:    store double [[CF64_REAL]], ptr [[BYVAL_TEMP_REALP]], align 8
 // CFINITEONLY-NEXT:    store double [[CF64_IMAG]], ptr [[BYVAL_TEMP_IMAGP]], align 8
-// CFINITEONLY-NEXT:    [[COERCE5_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[COERCE5]], i32 0, i32 0
-// CFINITEONLY-NEXT:    [[COERCE5_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[COERCE5]], i32 0, i32 1
-// CFINITEONLY-NEXT:    store half [[CF16_REAL]], ptr [[COERCE5_REALP]], align 2
-// CFINITEONLY-NEXT:    store half [[CF16_IMAG]], ptr [[COERCE5_IMAGP]], align 2
-// CFINITEONLY-NEXT:    [[TMP13:%.*]] = load <2 x half>, ptr [[COERCE5]], align 2
-// CFINITEONLY-NEXT:    [[CALL:%.*]] = call nnan ninf nofpclass(nan inf) float (float, ...) [[TMP2]](float noundef nofpclass(nan inf) [[TMP3]], double noundef nofpclass(nan inf) [[CONV]], double noundef nofpclass(nan inf) [[TMP5]], half noundef nofpclass(nan inf) [[TMP6]], double noundef nofpclass(nan inf) [[TMP10]], <2 x double> noundef nofpclass(nan inf) [[TMP8]], i32 noundef [[TMP11]], <2 x float> noundef nofpclass(nan inf) [[TMP12]], ptr noundef byval({ double, double }) align 8 [[BYVAL_TEMP]], <2 x half> noundef nofpclass(nan inf) [[TMP13]])
+// CFINITEONLY-NEXT:    [[BYVAL_TEMP3_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[BYVAL_TEMP3]], i32 0, i32 0
+// CFINITEONLY-NEXT:    [[BYVAL_TEMP3_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[BYVAL_TEMP3]], i32 0, i32 1
+// CFINITEONLY-NEXT:    store half [[CF16_REAL]], ptr [[BYVAL_TEMP3_REALP]], align 8
+// CFINITEONLY-NEXT:    store half [[CF16_IMAG]], ptr [[BYVAL_TEMP3_IMAGP]], align 2
+// CFINITEONLY-NEXT:    [[CALL:%.*]] = call nnan ninf nofpclass(nan inf) float (float, ...) [[TMP0]](float noundef nofpclass(nan inf) [[TMP1]], double noundef nofpclass(nan inf) [[CONV]], double noundef nofpclass(nan inf) [[TMP3]], half noundef nofpclass(nan inf) [[TMP4]], double noundef nofpclass(nan inf) [[TMP8]], <2 x double> noundef nofpclass(nan inf) [[TMP6]], <2 x half> noundef nofpclass(nan inf) [[TMP7]], <2 x float> noundef nofpclass(nan inf) [[TMP9]], ptr noundef byval({ double, double }) align 8 [[BYVAL_TEMP]], ptr noundef byval({ half, half }) align 8 [[BYVAL_TEMP3]])
 // CFINITEONLY-NEXT:    ret float [[CALL]]
 //
 // CLFINITEONLY: Function Attrs: convergent norecurse nounwind
 // CLFINITEONLY-LABEL: define dso_local nofpclass(nan inf) float @call_variadic_indirect
-// CLFINITEONLY-SAME: (ptr nofree noundef readonly captures(none) [[FPTR:%.*]], float noundef nofpclass(nan inf) [[F32:%.*]], double noundef nofpclass(nan inf) [[F64:%.*]], half noundef nofpclass(nan inf) [[F16:%.*]], double noundef nofpclass(nan inf) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(nan inf) [[V2F64:%.*]], i32 noundef [[V2F16_COERCE:%.*]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE:%.*]], double noundef nofpclass(nan inf) [[CF64_COERCE0:%.*]], double noundef nofpclass(nan inf) [[CF64_COERCE1:%.*]], ptr nofree noundef readonly byval({ half, half }) align 8 captures(none) [[CF16:%.*]]) local_unnamed_addr #[[ATTR5]] {
+// CLFINITEONLY-SAME: (ptr nofree noundef readonly captures(none) [[FPTR:%.*]], float noundef nofpclass(nan inf) [[F32:%.*]], double noundef nofpclass(nan inf) [[F64:%.*]], half noundef nofpclass(nan inf) [[F16:%.*]], double noundef nofpclass(nan inf) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(nan inf) [[V2F64:%.*]], <2 x half> noundef nofpclass(nan inf) [[V2F16:%.*]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE:%.*]], ptr nofree noundef readonly byval({ double, double }) align 8 captures(none) [[CF64:%.*]], <2 x half> noundef nofpclass(nan inf) [[CF16_COERCE:%.*]]) local_unnamed_addr #[[ATTR5]] {
 // CLFINITEONLY-NEXT:  entry:
 // CLFINITEONLY-NEXT:    [[BYVAL_TEMP:%.*]] = alloca { double, double }, align 8
+// CLFINITEONLY-NEXT:    [[BYVAL_TEMP3:%.*]] = alloca { half, half }, align 8
 // CLFINITEONLY-NEXT:    [[CONV:%.*]] = fpext nnan ninf float [[F32]] to double
-// CLFINITEONLY-NEXT:    [[CF16_REAL:%.*]] = load half, ptr [[CF16]], align 8
-// CLFINITEONLY-NEXT:    [[CF16_IMAGP:%.*]] = getelementptr inbounds nuw i8, ptr [[CF16]], i64 2
-// CLFINITEONLY-NEXT:    [[CF16_IMAG:%.*]] = load half, ptr [[CF16_IMAGP]], align 2
+// CLFINITEONLY-NEXT:    [[CF64_REAL:%.*]] = load double, ptr [[CF64]], align 8
+// CLFINITEONLY-NEXT:    [[CF64_IMAGP:%.*]] = getelementptr inbounds nuw i8, ptr [[CF64]], i64 8
+// CLFINITEONLY-NEXT:    [[CF64_IMAG:%.*]] = load double, ptr [[CF64_IMAGP]], align 8
+// CLFINITEONLY-NEXT:    [[CF16_SROA_0_0_VEC_EXTRACT:%.*]] = extractelement <2 x half> [[CF16_COERCE]], i64 0
+// CLFINITEONLY-NEXT:    [[CF16_SROA_0_2_VEC_EXTRACT:%.*]] = extractelement <2 x half> [[CF16_COERCE]], i64 1
 // CLFINITEONLY-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[BYVAL_TEMP]]) #[[ATTR12]]
 // CLFINITEONLY-NEXT:    [[BYVAL_TEMP_IMAGP:%.*]] = getelementptr inbounds nuw i8, ptr [[BYVAL_TEMP]], i64 8
-// CLFINITEONLY-NEXT:    store double [[CF64_COERCE0]], ptr [[BYVAL_TEMP]], align 8
-// CLFINITEONLY-NEXT:    store double [[CF64_COERCE1]], ptr [[BYVAL_TEMP_IMAGP]], align 8
-// CLFINITEONLY-NEXT:    [[COERCE5_SROA_0_0_VEC_INSERT:%.*]] = insertelement <2 x half> poison, half [[CF16_REAL]], i64 0
-// CLFINITEONLY-NEXT:    [[COERCE5_SROA_0_2_VEC_INSERT:%.*]] = insertelement <2 x half> [[COERCE5_SROA_0_0_VEC_INSERT]], half [[CF16_IMAG]], i64 1
-// CLFINITEONLY-NEXT:    [[CALL:%.*]] = tail call nnan ninf nofpclass(nan inf) float (float, ...) [[FPTR]](float noundef nofpclass(nan inf) [[F32]], double noundef nofpclass(nan inf) [[CONV]], double noundef nofpclass(nan inf) [[F64]], half noundef nofpclass(nan inf) [[F16]], double noundef nofpclass(nan inf) [[V2F32_COERCE]], <2 x double> noundef nofpclass(nan inf) [[V2F64]], i32 noundef [[V2F16_COERCE]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE]], ptr noundef nonnull byval({ double, double }) align 8 [[BYVAL_TEMP]], <2 x half> noundef nofpclass(nan inf) [[COERCE5_SROA_0_2_VEC_INSERT]]) #[[ATTR11]]
+// CLFINITEONLY-NEXT:    store double [[CF64_REAL]], ptr [[BYVAL_TEMP]], align 8
+// CLFINITEONLY-NEXT:    store double [[CF64_IMAG]], ptr [[BYVAL_TEMP_IMAGP]], align 8
+// CLFINITEONLY-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[BYVAL_TEMP3]]) #[[ATTR12]]
+// CLFINITEONLY-NEXT:    [[BYVAL_TEMP3_IMAGP:%.*]] = getelementptr inbounds nuw i8, ptr [[BYVAL_TEMP3]], i64 2
+// CLFINITEONLY-NEXT:    store half [[CF16_SROA_0_0_VEC_EXTRACT]], ptr [[BYVAL_TEMP3]], align 8
+// CLFINITEONLY-NEXT:    store half [[CF16_SROA_0_2_VEC_EXTRACT]], ptr [[BYVAL_TEMP3_IMAGP]], align 2
+// CLFINITEONLY-NEXT:    [[CALL:%.*]] = tail call nnan ninf nofpclass(nan inf) float (float, ...) [[FPTR]](float noundef nofpclass(nan inf) [[F32]], double noundef nofpclass(nan inf) [[CONV]], double noundef nofpclass(nan inf) [[F64]], half noundef nofpclass(nan inf) [[F16]], double noundef nofpclass(nan inf) [[V2F32_COERCE]], <2 x double> noundef nofpclass(nan inf) [[V2F64]], <2 x half> noundef nofpclass(nan inf) [[V2F16]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE]], ptr noundef nonnull byval({ double, double }) align 8 [[BYVAL_TEMP]], ptr noundef nonnull byval({ half, half }) align 8 [[BYVAL_TEMP3]]) #[[ATTR11]]
 // CLFINITEONLY-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[BYVAL_TEMP]]) #[[ATTR12]]
+// CLFINITEONLY-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[BYVAL_TEMP3]]) #[[ATTR12]]
 // CLFINITEONLY-NEXT:    ret float [[CALL]]
 //
 // NONANS: Function Attrs: noinline nounwind optnone
 // NONANS-LABEL: define dso_local nofpclass(nan) float @call_variadic_indirect
-// NONANS-SAME: (ptr noundef [[FPTR:%.*]], float noundef nofpclass(nan) [[F32:%.*]], double noundef nofpclass(nan) [[F64:%.*]], half noundef nofpclass(nan) [[F16:%.*]], double noundef nofpclass(nan) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(nan) [[V2F64:%.*]], i32 noundef [[V2F16_COERCE:%.*]], <2 x float> noundef nofpclass(nan) [[CF32_COERCE:%.*]], double noundef nofpclass(nan) [[CF64_COERCE0:%.*]], double noundef nofpclass(nan) [[CF64_COERCE1:%.*]], ptr noundef byval({ half, half }) align 8 [[CF16:%.*]]) #[[ATTR2]] {
+// NONANS-SAME: (ptr noundef [[FPTR:%.*]], float noundef nofpclass(nan) [[F32:%.*]], double noundef nofpclass(nan) [[F64:%.*]], half noundef nofpclass(nan) [[F16:%.*]], double noundef nofpclass(nan) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(nan) [[V2F64:%.*]], <2 x half> noundef nofpclass(nan) [[V2F16:%.*]], <2 x float> noundef nofpclass(nan) [[CF32_COERCE:%.*]], ptr noundef byval({ double, double }) align 8 [[CF64:%.*]], <2 x half> noundef nofpclass(nan) [[CF16_COERCE:%.*]]) #[[ATTR2]] {
 // NONANS-NEXT:  entry:
 // NONANS-NEXT:    [[V2F32:%.*]] = alloca <2 x float>, align 8
-// NONANS-NEXT:    [[V2F16:%.*]] = alloca <2 x half>, align 4
 // NONANS-NEXT:    [[CF32:%.*]] = alloca { float, float }, align 4
-// NONANS-NEXT:    [[CF64:%.*]] = alloca { double, double }, align 8
+// NONANS-NEXT:    [[CF16:%.*]] = alloca { half, half }, align 2
 // NONANS-NEXT:    [[FPTR_ADDR:%.*]] = alloca ptr, align 8
 // NONANS-NEXT:    [[F32_ADDR:%.*]] = alloca float, align 4
 // NONANS-NEXT:    [[F64_ADDR:%.*]] = alloca double, align 8
@@ -1206,35 +1159,29 @@ float call_variadic(float f32, double f64, _Float16 f16,
 // NONANS-NEXT:    [[V2F64_ADDR:%.*]] = alloca <2 x double>, align 16
 // NONANS-NEXT:    [[V2F16_ADDR:%.*]] = alloca <2 x half>, align 4
 // NONANS-NEXT:    [[COERCE:%.*]] = alloca <2 x float>, align 8
-// NONANS-NEXT:    [[COERCE3:%.*]] = alloca <2 x half>, align 4
-// NONANS-NEXT:    [[COERCE4:%.*]] = alloca { float, float }, align 4
+// NONANS-NEXT:    [[COERCE2:%.*]] = alloca { float, float }, align 4
 // NONANS-NEXT:    [[BYVAL_TEMP:%.*]] = alloca { double, double }, align 8
-// NONANS-NEXT:    [[COERCE5:%.*]] = alloca { half, half }, align 2
+// NONANS-NEXT:    [[BYVAL_TEMP3:%.*]] = alloca { half, half }, align 8
 // NONANS-NEXT:    store double [[V2F32_COERCE]], ptr [[V2F32]], align 8
 // NONANS-NEXT:    [[V2F321:%.*]] = load <2 x float>, ptr [[V2F32]], align 8
-// NONANS-NEXT:    store i32 [[V2F16_COERCE]], ptr [[V2F16]], align 4
-// NONANS-NEXT:    [[V2F162:%.*]] = load <2 x half>, ptr [[V2F16]], align 4
 // NONANS-NEXT:    store <2 x float> [[CF32_COERCE]], ptr [[CF32]], align 4
-// NONANS-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[CF64]], i32 0, i32 0
-// NONANS-NEXT:    store double [[CF64_COERCE0]], ptr [[TMP0]], align 8
-// NONANS-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[CF64]], i32 0, i32 1
-// NONANS-NEXT:    store double [[CF64_COERCE1]], ptr [[TMP1]], align 8
+// NONANS-NEXT:    store <2 x half> [[CF16_COERCE]], ptr [[CF16]], align 2
 // NONANS-NEXT:    store ptr [[FPTR]], ptr [[FPTR_ADDR]], align 8
 // NONANS-NEXT:    store float [[F32]], ptr [[F32_ADDR]], align 4
 // NONANS-NEXT:    store double [[F64]], ptr [[F64_ADDR]], align 8
 // NONANS-NEXT:    store half [[F16]], ptr [[F16_ADDR]], align 2
 // NONANS-NEXT:    store <2 x float> [[V2F321]], ptr [[V2F32_ADDR]], align 8
 // NONANS-NEXT:    store <2 x double> [[V2F64]], ptr [[V2F64_ADDR]], align 16
-// NONANS-NEXT:    store <2 x half> [[V2F162]], ptr [[V2F16_ADDR]], align 4
-// NONANS-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[FPTR_ADDR]], align 8
-// NONANS-NEXT:    [[TMP3:%.*]] = load float, ptr [[F32_ADDR]], align 4
-// NONANS-NEXT:    [[TMP4:%.*]] = load float, ptr [[F32_ADDR]], align 4
-// NONANS-NEXT:    [[CONV:%.*]] = fpext nnan float [[TMP4]] to double
-// NONANS-NEXT:    [[TMP5:%.*]] = load double, ptr [[F64_ADDR]], align 8
-// NONANS-NEXT:    [[TMP6:%.*]] = load half, ptr [[F16_ADDR]], align 2
-// NONANS-NEXT:    [[TMP7:%.*]] = load <2 x float>, ptr [[V2F32_ADDR]], align 8
-// NONANS-NEXT:    [[TMP8:%.*]] = load <2 x double>, ptr [[V2F64_ADDR]], align 16
-// NONANS-NEXT:    [[TMP9:%.*]] = load <2 x half>, ptr [[V2F16_ADDR]], align 4
+// NONANS-NEXT:    store <2 x half> [[V2F16]], ptr [[V2F16_ADDR]], align 4
+// NONANS-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[FPTR_ADDR]], align 8
+// NONANS-NEXT:    [[TMP1:%.*]] = load float, ptr [[F32_ADDR]], align 4
+// NONANS-NEXT:    [[TMP2:%.*]] = load float, ptr [[F32_ADDR]], align 4
+// NONANS-NEXT:    [[CONV:%.*]] = fpext nnan float [[TMP2]] to double
+// NONANS-NEXT:    [[TMP3:%.*]] = load double, ptr [[F64_ADDR]], align 8
+// NONANS-NEXT:    [[TMP4:%.*]] = load half, ptr [[F16_ADDR]], align 2
+// NONANS-NEXT:    [[TMP5:%.*]] = load <2 x float>, ptr [[V2F32_ADDR]], align 8
+// NONANS-NEXT:    [[TMP6:%.*]] = load <2 x double>, ptr [[V2F64_ADDR]], align 16
+// NONANS-NEXT:    [[TMP7:%.*]] = load <2 x half>, ptr [[V2F16_ADDR]], align 4
 // NONANS-NEXT:    [[CF32_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[CF32]], i32 0, i32 0
 // NONANS-NEXT:    [[CF32_REAL:%.*]] = load float, ptr [[CF32_REALP]], align 4
 // NONANS-NEXT:    [[CF32_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[CF32]], i32 0, i32 1
@@ -1244,38 +1191,34 @@ float call_variadic(float f32, double f64, _Float16 f16,
 // NONANS-NEXT:    [[CF64_IMAGP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[CF64]], i32 0, i32 1
 // NONANS-NEXT:    [[CF64_IMAG:%.*]] = load double, ptr [[CF64_IMAGP]], align 8
 // NONANS-NEXT:    [[CF16_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[CF16]], i32 0, i32 0
-// NONANS-NEXT:    [[CF16_REAL:%.*]] = load half, ptr [[CF16_REALP]], align 8
+// NONANS-NEXT:    [[CF16_REAL:%.*]] = load half, ptr [[CF16_REALP]], align 2
 // NONANS-NEXT:    [[CF16_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[CF16]], i32 0, i32 1
 // NONANS-NEXT:    [[CF16_IMAG:%.*]] = load half, ptr [[CF16_IMAGP]], align 2
-// NONANS-NEXT:    store <2 x float> [[TMP7]], ptr [[COERCE]], align 8
-// NONANS-NEXT:    [[TMP10:%.*]] = load double, ptr [[COERCE]], align 8
-// NONANS-NEXT:    store <2 x half> [[TMP9]], ptr [[COERCE3]], align 4
-// NONANS-NEXT:    [[TMP11:%.*]] = load i32, ptr [[COERCE3]], align 4
-// NONANS-NEXT:    [[COERCE4_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE4]], i32 0, i32 0
-// NONANS-NEXT:    [[COERCE4_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE4]], i32 0, i32 1
-// NONANS-NEXT:    store float [[CF32_REAL]], ptr [[COERCE4_REALP]], align 4
-// NONANS-NEXT:    store float [[CF32_IMAG]], ptr [[COERCE4_IMAGP]], align 4
-// NONANS-NEXT:    [[TMP12:%.*]] = load <2 x float>, ptr [[COERCE4]], align 4
+// NONANS-NEXT:    store <2 x float> [[TMP5]], ptr [[COERCE]], align 8
+// NONANS-NEXT:    [[TMP8:%.*]] = load double, ptr [[COERCE]], align 8
+// NONANS-NEXT:    [[COERCE2_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE2]], i32 0, i32 0
+// NONANS-NEXT:    [[COERCE2_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE2]], i32 0, i32 1
+// NONANS-NEXT:    store float [[CF32_REAL]], ptr [[COERCE2_REALP]], align 4
+// NONANS-NEXT:    store float [[CF32_IMAG]], ptr [[COERCE2_IMAGP]], align 4
+// NONANS-NEXT:    [[TMP9:%.*]] = load <2 x float>, ptr [[COERCE2]], align 4
 // NONANS-NEXT:    [[BYVAL_TEMP_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[BYVAL_TEMP]], i32 0, i32 0
 // NONANS-NEXT:    [[BYVAL_TEMP_IMAGP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[BYVAL_TEMP]], i32 0, i32 1
 // NONANS-NEXT:    store double [[CF64_REAL]], ptr [[BYVAL_TEMP_REALP]], align 8
 // NONANS-NEXT:    store double [[CF64_IMAG]], ptr [[BYVAL_TEMP_IMAGP]], align 8
-// NONANS-NEXT:    [[COERCE5_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[COERCE5]], i32 0, i32 0
-// NONANS-NEXT:    [[COERCE5_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[COERCE5]], i32 0, i32 1
-// NONANS-NEXT:    store half [[CF16_REAL]], ptr [[COERCE5_REALP]], align 2
-// NONANS-NEXT:    store half [[CF16_IMAG]], ptr [[COERCE5_IMAGP]], align 2
-// NONANS-NEXT:    [[TMP13:%.*]] = load <2 x half>, ptr [[COERCE5]], align 2
-// NONANS-NEXT:    [[CALL:%.*]] = call nnan nofpclass(nan) float (float, ...) [[TMP2]](float noundef nofpclass(nan) [[TMP3]], double noundef nofpclass(nan) [[CONV]], double noundef nofpclass(nan) [[TMP5]], half noundef nofpclass(nan) [[TMP6]], double noundef nofpclass(nan) [[TMP10]], <2 x double> noundef nofpclass(nan) [[TMP8]], i32 noundef [[TMP11]], <2 x float> noundef nofpclass(nan) [[TMP12]], ptr noundef byval({ double, double }) align 8 [[BYVAL_TEMP]], <2 x half> noundef nofpclass(nan) [[TMP13]])
+// NONANS-NEXT:    [[BYVAL_TEMP3_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[BYVAL_TEMP3]], i32 0, i32 0
+// NONANS-NEXT:    [[BYVAL_TEMP3_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[BYVAL_TEMP3]], i32 0, i32 1
+// NONANS-NEXT:    store half [[CF16_REAL]], ptr [[BYVAL_TEMP3_REALP]], align 8
+// NONANS-NEXT:    store half [[CF16_IMAG]], ptr [[BYVAL_TEMP3_IMAGP]], align 2
+// NONANS-NEXT:    [[CALL:%.*]] = call nnan nofpclass(nan) float (float, ...) [[TMP0]](float noundef nofpclass(nan) [[TMP1]], double noundef nofpclass(nan) [[CONV]], double noundef nofpclass(nan) [[TMP3]], half noundef nofpclass(nan) [[TMP4]], double noundef nofpclass(nan) [[TMP8]], <2 x double> noundef nofpclass(nan) [[TMP6]], <2 x half> noundef nofpclass(nan) [[TMP7]], <2 x float> noundef nofpclass(nan) [[TMP9]], ptr noundef byval({ double, double }) align 8 [[BYVAL_TEMP]], ptr noundef byval({ half, half }) align 8 [[BYVAL_TEMP3]])
 // NONANS-NEXT:    ret float [[CALL]]
 //
 // NOINFS: Function Attrs: noinline nounwind optnone
 // NOINFS-LABEL: define dso_local nofpclass(inf) float @call_variadic_indirect
-// NOINFS-SAME: (ptr noundef [[FPTR:%.*]], float noundef nofpclass(inf) [[F32:%.*]], double noundef nofpclass(inf) [[F64:%.*]], half noundef nofpclass(inf) [[F16:%.*]], double noundef nofpclass(inf) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(inf) [[V2F64:%.*]], i32 noundef [[V2F16_COERCE:%.*]], <2 x float> noundef nofpclass(inf) [[CF32_COERCE:%.*]], double noundef nofpclass(inf) [[CF64_COERCE0:%.*]], double noundef nofpclass(inf) [[CF64_COERCE1:%.*]], ptr noundef byval({ half, half }) align 8 [[CF16:%.*]]) #[[ATTR2]] {
+// NOINFS-SAME: (ptr noundef [[FPTR:%.*]], float noundef nofpclass(inf) [[F32:%.*]], double noundef nofpclass(inf) [[F64:%.*]], half noundef nofpclass(inf) [[F16:%.*]], double noundef nofpclass(inf) [[V2F32_COERCE:%.*]], <2 x double> noundef nofpclass(inf) [[V2F64:%.*]], <2 x half> noundef nofpclass(inf) [[V2F16:%.*]], <2 x float> noundef nofpclass(inf) [[CF32_COERCE:%.*]], ptr noundef byval({ double, double }) align 8 [[CF64:%.*]], <2 x half> noundef nofpclass(inf) [[CF16_COERCE:%.*]]) #[[ATTR2]] {
 // NOINFS-NEXT:  entry:
 // NOINFS-NEXT:    [[V2F32:%.*]] = alloca <2 x float>, align 8
-// NOINFS-NEXT:    [[V2F16:%.*]] = alloca <2 x half>, align 4
 // NOINFS-NEXT:    [[CF32:%.*]] = alloca { float, float }, align 4
-// NOINFS-NEXT:    [[CF64:%.*]] = alloca { double, double }, align 8
+// NOINFS-NEXT:    [[CF16:%.*]] = alloca { half, half }, align 2
 // NOINFS-NEXT:    [[FPTR_ADDR:%.*]] = alloca ptr, align 8
 // NOINFS-NEXT:    [[F32_ADDR:%.*]] = alloca float, align 4
 // NOINFS-NEXT:    [[F64_ADDR:%.*]] = alloca double, align 8
@@ -1284,35 +1227,29 @@ float call_variadic(float f32, double f64, _Float16 f16,
 // NOINFS-NEXT:    [[V2F64_ADDR:%.*]] = alloca <2 x double>, align 16
 // NOINFS-NEXT:    [[V2F16_ADDR:%.*]] = alloca <2 x half>, align 4
 // NOINFS-NEXT:    [[COERCE:%.*]] = alloca <2 x float>, align 8
-// NOINFS-NEXT:    [[COERCE3:%.*]] = alloca <2 x half>, align 4
-// NOINFS-NEXT:    [[COERCE4:%.*]] = alloca { float, float }, align 4
+// NOINFS-NEXT:    [[COERCE2:%.*]] = alloca { float, float }, align 4
 // NOINFS-NEXT:    [[BYVAL_TEMP:%.*]] = alloca { double, double }, align 8
-// NOINFS-NEXT:    [[COERCE5:%.*]] = alloca { half, half }, align 2
+// NOINFS-NEXT:    [[BYVAL_TEMP3:%.*]] = alloca { half, half }, align 8
 // NOINFS-NEXT:    store double [[V2F32_COERCE]], ptr [[V2F32]], align 8
 // NOINFS-NEXT:    [[V2F321:%.*]] = load <2 x float>, ptr [[V2F32]], align 8
-// NOINFS-NEXT:    store i32 [[V2F16_COERCE]], ptr [[V2F16]], align 4
-// NOINFS-NEXT:    [[V2F162:%.*]] = load <2 x half>, ptr [[V2F16]], align 4
 // NOINFS-NEXT:    store <2 x float> [[CF32_COERCE]], ptr [[CF32]], align 4
-// NOINFS-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[CF64]], i32 0, i32 0
-// NOINFS-NEXT:    store double [[CF64_COERCE0]], ptr [[TMP0]], align 8
-// NOINFS-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[CF64]], i32 0, i32 1
-// NOINFS-NEXT:    store double [[CF64_COERCE1]], ptr [[TMP1]], align 8
+// NOINFS-NEXT:    store <2 x half> [[CF16_COERCE]], ptr [[CF16]], align 2
 // NOINFS-NEXT:    store ptr [[FPTR]], ptr [[FPTR_ADDR]], align 8
 // NOINFS-NEXT:    store float [[F32]], ptr [[F32_ADDR]], align 4
 // NOINFS-NEXT:    store double [[F64]], ptr [[F64_ADDR]], align 8
 // NOINFS-NEXT:    store half [[F16]], ptr [[F16_ADDR]], align 2
 // NOINFS-NEXT:    store <2 x float> [[V2F321]], ptr [[V2F32_ADDR]], align 8
 // NOINFS-NEXT:    store <2 x double> [[V2F64]], ptr [[V2F64_ADDR]], align 16
-// NOINFS-NEXT:    store <2 x half> [[V2F162]], ptr [[V2F16_ADDR]], align 4
-// NOINFS-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[FPTR_ADDR]], align 8
-// NOINFS-NEXT:    [[TMP3:%.*]] = load float, ptr [[F32_ADDR]], align 4
-// NOINFS-NEXT:    [[TMP4:%.*]] = load float, ptr [[F32_ADDR]], align 4
-// NOINFS-NEXT:    [[CONV:%.*]] = fpext ninf float [[TMP4]] to double
-// NOINFS-NEXT:    [[TMP5:%.*]] = load double, ptr [[F64_ADDR]], align 8
-// NOINFS-NEXT:    [[TMP6:%.*]] = load half, ptr [[F16_ADDR]], align 2
-// NOINFS-NEXT:    [[TMP7:%.*]] = load <2 x float>, ptr [[V2F32_ADDR]], align 8
-// NOINFS-NEXT:    [[TMP8:%.*]] = load <2 x double>, ptr [[V2F64_ADDR]], align 16
-// NOINFS-NEXT:    [[TMP9:%.*]] = load <2 x half>, ptr [[V2F16_ADDR]], align 4
+// NOINFS-NEXT:    store <2 x half> [[V2F16]], ptr [[V2F16_ADDR]], align 4
+// NOINFS-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[FPTR_ADDR]], align 8
+// NOINFS-NEXT:    [[TMP1:%.*]] = load float, ptr [[F32_ADDR]], align 4
+// NOINFS-NEXT:    [[TMP2:%.*]] = load float, ptr [[F32_ADDR]], align 4
+// NOINFS-NEXT:    [[CONV:%.*]] = fpext ninf float [[TMP2]] to double
+// NOINFS-NEXT:    [[TMP3:%.*]] = load double, ptr [[F64_ADDR]], align 8
+// NOINFS-NEXT:    [[TMP4:%.*]] = load half, ptr [[F16_ADDR]], align 2
+// NOINFS-NEXT:    [[TMP5:%.*]] = load <2 x float>, ptr [[V2F32_ADDR]], align 8
+// NOINFS-NEXT:    [[TMP6:%.*]] = load <2 x double>, ptr [[V2F64_ADDR]], align 16
+// NOINFS-NEXT:    [[TMP7:%.*]] = load <2 x half>, ptr [[V2F16_ADDR]], align 4
 // NOINFS-NEXT:    [[CF32_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[CF32]], i32 0, i32 0
 // NOINFS-NEXT:    [[CF32_REAL:%.*]] = load float, ptr [[CF32_REALP]], align 4
 // NOINFS-NEXT:    [[CF32_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[CF32]], i32 0, i32 1
@@ -1322,28 +1259,25 @@ float call_variadic(float f32, double f64, _Float16 f16,
 // NOINFS-NEXT:    [[CF64_IMAGP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[CF64]], i32 0, i32 1
 // NOINFS-NEXT:    [[CF64_IMAG:%.*]] = load double, ptr [[CF64_IMAGP]], align 8
 // NOINFS-NEXT:    [[CF16_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[CF16]], i32 0, i32 0
-// NOINFS-NEXT:    [[CF16_REAL:%.*]] = load half, ptr [[CF16_REALP]], align 8
+// NOINFS-NEXT:    [[CF16_REAL:%.*]] = load half, ptr [[CF16_REALP]], align 2
 // NOINFS-NEXT:    [[CF16_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[CF16]], i32 0, i32 1
 // NOINFS-NEXT:    [[CF16_IMAG:%.*]] = load half, ptr [[CF16_IMAGP]], align 2
-// NOINFS-NEXT:    store <2 x float> [[TMP7]], ptr [[COERCE]], align 8
-// NOINFS-NEXT:    [[TMP10:%.*]] = load double, ptr [[COERCE]], align 8
-// NOINFS-NEXT:    store <2 x half> [[TMP9]], ptr [[COERCE3]], align 4
-// NOINFS-NEXT:    [[TMP11:%.*]] = load i32, ptr [[COERCE3]], align 4
-// NOINFS-NEXT:    [[COERCE4_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE4]], i32 0, i32 0
-// NOINFS-NEXT:    [[COERCE4_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE4]], i32 0, i32 1
-// NOINFS-NEXT:    store float [[CF32_REAL]], ptr [[COERCE4_REALP]], align 4
-// NOINFS-NEXT:    store float [[CF32_IMAG]], ptr [[COERCE4_IMAGP]], align 4
-// NOINFS-NEXT:    [[TMP12:%.*]] = load <2 x float>, ptr [[COERCE4]], align 4
+// NOINFS-NEXT:    store <2 x float> [[TMP5]], ptr [[COERCE]], align 8
+// NOINFS-NEXT:    [[TMP8:%.*]] = load double, ptr [[COERCE]], align 8
+// NOINFS-NEXT:    [[COERCE2_REALP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE2]], i32 0, i32 0
+// NOINFS-NEXT:    [[COERCE2_IMAGP:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[COERCE2]], i32 0, i32 1
+// NOINFS-NEXT:    store float [[CF32_REAL]], ptr [[COERCE2_REALP]], align 4
+// NOINFS-NEXT:    store float [[CF32_IMAG]], ptr [[COERCE2_IMAGP]], align 4
+// NOINFS-NEXT:    [[TMP9:%.*]] = load <2 x float>, ptr [[COERCE2]], align 4
 // NOINFS-NEXT:    [[BYVAL_TEMP_REALP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[BYVAL_TEMP]], i32 0, i32 0
 // NOINFS-NEXT:    [[BYVAL_TEMP_IMAGP:%.*]] = getelementptr inbounds nuw { double, double }, ptr [[BYVAL_TEMP]], i32 0, i32 1
 // NOINFS-NEXT:    store double [[CF64_REAL]], ptr [[BYVAL_TEMP_REALP]], align 8
 // NOINFS-NEXT:    store double [[CF64_IMAG]], ptr [[BYVAL_TEMP_IMAGP]], align 8
-// NOINFS-NEXT:    [[COERCE5_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[COERCE5]], i32 0, i32 0
-// NOINFS-NEXT:    [[COERCE5_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[COERCE5]], i32 0, i32 1
-// NOINFS-NEXT:    store half [[CF16_REAL]], ptr [[COERCE5_REALP]], align 2
-// NOINFS-NEXT:    store half [[CF16_IMAG]], ptr [[COERCE5_IMAGP]], align 2
-// NOINFS-NEXT:    [[TMP13:%.*]] = load <2 x half>, ptr [[COERCE5]], align 2
-// NOINFS-NEXT:    [[CALL:%.*]] = call ninf nofpclass(inf) float (float, ...) [[TMP2]](float noundef nofpclass(inf) [[TMP3]], double noundef nofpclass(inf) [[CONV]], double noundef nofpclass(inf) [[TMP5]], half noundef nofpclass(inf) [[TMP6]], double noundef nofpclass(inf) [[TMP10]], <2 x double> noundef nofpclass(inf) [[TMP8]], i32 noundef [[TMP11]], <2 x float> noundef nofpclass(inf) [[TMP12]], ptr noundef byval({ double, double }) align 8 [[BYVAL_TEMP]], <2 x half> noundef nofpclass(inf) [[TMP13]])
+// NOINFS-NEXT:    [[BYVAL_TEMP3_REALP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[BYVAL_TEMP3]], i32 0, i32 0
+// NOINFS-NEXT:    [[BYVAL_TEMP3_IMAGP:%.*]] = getelementptr inbounds nuw { half, half }, ptr [[BYVAL_TEMP3]], i32 0, i32 1
+// NOINFS-NEXT:    store half [[CF16_REAL]], ptr [[BYVAL_TEMP3_REALP]], align 8
+// NOINFS-NEXT:    store half [[CF16_IMAG]], ptr [[BYVAL_TEMP3_IMAGP]], align 2
+// NOINFS-NEXT:    [[CALL:%.*]] = call ninf nofpclass(inf) float (float, ...) [[TMP0]](float noundef nofpclass(inf) [[TMP1]], double noundef nofpclass(inf) [[CONV]], double noundef nofpclass(inf) [[TMP3]], half noundef nofpclass(inf) [[TMP4]], double noundef nofpclass(inf) [[TMP8]], <2 x double> noundef nofpclass(inf) [[TMP6]], <2 x half> noundef nofpclass(inf) [[TMP7]], <2 x float> noundef nofpclass(inf) [[TMP9]], ptr noundef byval({ double, double }) align 8 [[BYVAL_TEMP]], ptr noundef byval({ half, half }) align 8 [[BYVAL_TEMP3]])
 // NOINFS-NEXT:    ret float [[CALL]]
 //
 float call_variadic_indirect(float fptr(float, ...), float f32, double f64, _Float16 f16,
diff --git a/clang/test/CodeGen/sysv_abi.c b/clang/test/CodeGen/sysv_abi.c
index 81d2b26a096294..ed7a1e0f5ab71c 100644
--- a/clang/test/CodeGen/sysv_abi.c
+++ b/clang/test/CodeGen/sysv_abi.c
@@ -1,8 +1,10 @@
-// RUN: %clang_cc1 -triple x86_64-pc-win32 -emit-llvm  -target-cpu skylake-avx512 < %s | FileCheck %s --check-prefixes=CHECK,AVX
-// RUN: %clang_cc1 -triple x86_64-mingw    -emit-llvm  -target-cpu skylake-avx512 < %s | FileCheck %s --check-prefixes=CHECK,AVX
-// RUN: %clang_cc1 -triple x86_64-cygwin   -emit-llvm  -target-cpu skylake-avx512 < %s | FileCheck %s --check-prefixes=CHECK,AVX
-// RUN: %clang_cc1 -triple x86_64-linux -emit-llvm  -target-cpu skylake-avx512 < %s | FileCheck %s --check-prefixes=CHECK,AVX
-// RUN: %clang_cc1 -triple x86_64-uefi -emit-llvm  -target-cpu skylake-avx512 < %s | FileCheck %s --check-prefixes=CHECK,AVX
+// RUN: %clang_cc1 -triple x86_64-pc-win32 -emit-llvm  -target-cpu skylake-avx512 < %s | FileCheck %s --check-prefixes=CHECK,AVX512
+// RUN: %clang_cc1 -triple x86_64-mingw    -emit-llvm  -target-cpu skylake-avx512 < %s | FileCheck %s --check-prefixes=CHECK,AVX512
+// RUN: %clang_cc1 -triple x86_64-cygwin   -emit-llvm  -target-cpu skylake-avx512 < %s | FileCheck %s --check-prefixes=CHECK,AVX512
+// RUN: %clang_cc1 -triple x86_64-linux -emit-llvm  -target-cpu skylake-avx512 < %s | FileCheck %s --check-prefixes=CHECK,AVX512
+// RUN: %clang_cc1 -triple x86_64-uefi -emit-llvm  -target-cpu skylake-avx512 < %s | FileCheck %s --check-prefixes=CHECK,AVX512
+// Note: x86-64-v3 implies support for AVX2 but not AVX512
+// RUN: %clang_cc1 -triple x86_64-linux -emit-llvm -target-cpu x86-64-v3 < %s | FileCheck %s --check-prefixes=CHECK,AVX2
 // RUN: %clang_cc1 -triple x86_64-pc-win32 -emit-llvm < %s | FileCheck %s --check-prefixes=CHECK,NOAVX
 // RUN: %clang_cc1 -triple x86_64-mingw    -emit-llvm < %s | FileCheck %s --check-prefixes=CHECK,NOAVX
 // RUN: %clang_cc1 -triple x86_64-cygwin   -emit-llvm < %s | FileCheck %s --check-prefixes=CHECK,NOAVX
@@ -45,13 +47,17 @@ void use_vectors(void) {
 }
 
 // CHECK: define {{(dso_local )?}}void @use_vectors()
-// AVX: call {{(x86_64_sysvcc )?}}<8 x float> @get_m256()
-// AVX: call {{(x86_64_sysvcc )?}}void @take_m256(<8 x float> noundef %{{.*}})
-// AVX: call {{(x86_64_sysvcc )?}}<16 x float> @get_m512()
-// AVX: call {{(x86_64_sysvcc )?}}void @take_m512(<16 x float> noundef %{{.*}})
-// NOAVX: call {{(x86_64_sysvcc )?}}<8 x float> @get_m256()
+// AVX512: call {{(x86_64_sysvcc )?}}<8 x float> @get_m256()
+// AVX512: call {{(x86_64_sysvcc )?}}void @take_m256(<8 x float> noundef %{{.*}})
+// AVX512: call {{(x86_64_sysvcc )?}}<16 x float> @get_m512()
+// AVX512: call {{(x86_64_sysvcc )?}}void @take_m512(<16 x float> noundef %{{.*}})
+// AVX2: call {{(x86_64_sysvcc )?}}<8 x float> @get_m256()
+// AVX2: call {{(x86_64_sysvcc )?}}void @take_m256(<8 x float> noundef %{{.*}})
+// AVX2: call {{(x86_64_sysvcc )?}}void @get_m512(ptr {{[^,]*}} sret(<16 x float>) align 64 %{{.*}})
+// AVX2: call {{(x86_64_sysvcc )?}}void @take_m512(ptr noundef byval(<16 x float>) align 64 %{{.*}})
+// NOAVX: call {{(x86_64_sysvcc )?}}void @get_m256(ptr {{[^,]*}} sret(<8 x float>) align 32 %{{.*}})
 // NOAVX: call {{(x86_64_sysvcc )?}}void @take_m256(ptr noundef byval(<8 x float>) align 32 %{{.*}})
-// NOAVX: call {{(x86_64_sysvcc )?}}<16 x float> @get_m512()
+// NOAVX: call {{(x86_64_sysvcc )?}}void @get_m512(ptr {{[^,]*}} sret(<16 x float>) align 64 %{{.*}})
 // NOAVX: call {{(x86_64_sysvcc )?}}void @take_m512(ptr noundef byval(<16 x float>) align 64 %{{.*}})
 
 // Added test to explicitly cover the case when __attribute__((target("avx"))) is used 
diff --git a/clang/test/CodeGenCXX/ext-int-vector-abi.cpp b/clang/test/CodeGenCXX/ext-int-vector-abi.cpp
index f9d864d6217cdd..a5cffde03fc2fc 100644
--- a/clang/test/CodeGenCXX/ext-int-vector-abi.cpp
+++ b/clang/test/CodeGenCXX/ext-int-vector-abi.cpp
@@ -61,7 +61,7 @@ uint32_t4s ManglingTestRetParam(uint32_t4s Param) {
 
 typedef unsigned _BitInt(64) uint64_t4 __attribute__((ext_vector_type(4)));
 uint64_t4 ManglingTestRetParam(uint64_t4 Param) {
-// LIN64: define{{.*}} <4 x i64> @_Z20ManglingTestRetParamDv4_DU64_(ptr byval(<4 x i64>) align 32 %
+// LIN64: define{{.*}} void @_Z20ManglingTestRetParamDv4_DU64_(ptr{{.*}}sret(<4 x i64>) align 32 %{{.*}}, ptr byval(<4 x i64>) align 32 %
 // LIN32: define{{.*}} <4 x i64> @_Z20ManglingTestRetParamDv4_DU64_(<4 x i64> %
 // WIN64: define dso_local <4 x i64> @"?ManglingTestRetParam@@YAT?$__vector at U?$_UBitInt@$0EA@@__clang@@$03 at __clang@@T12@@Z"(<4 x i64> %
 // WIN32: define dso_local <4 x i64> @"?ManglingTestRetParam@@YAT?$__vector at U?$_UBitInt@$0EA@@__clang@@$03 at __clang@@T12@@Z"(<4 x i64> inreg %
@@ -70,7 +70,7 @@ uint64_t4 ManglingTestRetParam(uint64_t4 Param) {
 
 typedef unsigned long long uint64_t4s __attribute__((ext_vector_type(4)));
 uint64_t4s ManglingTestRetParam(uint64_t4s Param) {
-// LIN64: define{{.*}} <4 x i64> @_Z20ManglingTestRetParamDv4_y(ptr byval(<4 x i64>) align 32 %
+// LIN64: define{{.*}} void @_Z20ManglingTestRetParamDv4_y(ptr{{.*}}sret(<4 x i64>) align 32 %{{.*}}, ptr byval(<4 x i64>) align 32 %
 // LIN32: define{{.*}} <4 x i64> @_Z20ManglingTestRetParamDv4_y(<4 x i64> %
 // WIN64: define dso_local <4 x i64> @"?ManglingTestRetParam@@YAT?$__vector at _K$03 at __clang@@T12@@Z"(<4 x i64> %
 // WIN32: define dso_local <4 x i64> @"?ManglingTestRetParam@@YAT?$__vector at _K$03 at __clang@@T12@@Z"(<4 x i64> inreg %
@@ -79,7 +79,7 @@ uint64_t4s ManglingTestRetParam(uint64_t4s Param) {
 
 typedef _BitInt(32) vint32_t8 __attribute__((vector_size(32)));
 vint32_t8 ManglingTestRetParam(vint32_t8 Param) {
-// LIN64: define{{.*}} <8 x i32> @_Z20ManglingTestRetParamDv8_DB32_(ptr byval(<8 x i32>) align 32 %
+// LIN64: define{{.*}} void @_Z20ManglingTestRetParamDv8_DB32_(ptr{{.*}}sret(<8 x i32>) align 32 %{{.*}}, ptr byval(<8 x i32>) align 32 %
 // LIN32: define{{.*}} <8 x i32> @_Z20ManglingTestRetParamDv8_DB32_(<8 x i32> %
 // WIN64: define dso_local <8 x i32> @"?ManglingTestRetParam@@YA?AT?$__vector at U?$_BitInt@$0CA@@__clang@@$07 at __clang@@T12@@Z"(<8 x i32> %
 // WIN32: define dso_local <8 x i32> @"?ManglingTestRetParam@@YA?AT?$__vector at U?$_BitInt@$0CA@@__clang@@$07 at __clang@@T12@@Z"(<8 x i32> inreg %
@@ -88,7 +88,7 @@ vint32_t8 ManglingTestRetParam(vint32_t8 Param) {
 
 typedef int vint32_t8i __attribute__((vector_size(32)));
 vint32_t8i ManglingTestRetParam(vint32_t8i Param) {
-// LIN64: define{{.*}} <8 x i32> @_Z20ManglingTestRetParamDv8_i(ptr byval(<8 x i32>) align 32 %
+// LIN64: define{{.*}} void @_Z20ManglingTestRetParamDv8_i(ptr{{.*}}sret(<8 x i32>) align 32 %{{.*}}, ptr byval(<8 x i32>) align 32 %
 // LIN32: define{{.*}} <8 x i32> @_Z20ManglingTestRetParamDv8_i(<8 x i32> %
 // WIN64: define dso_local <8 x i32> @"?ManglingTestRetParam@@YA?AT?$__vector at H$07 at __clang@@T12@@Z"(<8 x i32> %
 // WIN32: define dso_local <8 x i32> @"?ManglingTestRetParam@@YA?AT?$__vector at H$07 at __clang@@T12@@Z"(<8 x i32> inreg %
diff --git a/clang/test/CodeGenCXX/matrix-vector-bit-int.cpp b/clang/test/CodeGenCXX/matrix-vector-bit-int.cpp
index 4be1cb3067c2fc..f5eaf7ba0207e8 100644
--- a/clang/test/CodeGenCXX/matrix-vector-bit-int.cpp
+++ b/clang/test/CodeGenCXX/matrix-vector-bit-int.cpp
@@ -51,8 +51,8 @@ i32x3 v2(i32x3 a) {
   return a + a;
 }
 
-// CHECK-LABEL: define dso_local noundef <3 x i512> @_Z2v3Dv3_DB512_(
-// CHECK-SAME: ptr noundef byval(<3 x i512>) align 256 [[TMP0:%.*]]) #[[ATTR2:[0-9]+]] {
+// CHECK-LABEL: define dso_local void @_Z2v3Dv3_DB512_(
+// CHECK-SAME: ptr dead_on_unwind noalias writable sret(<3 x i512>) align 256 [[AGG_RESULT:%.*]], ptr noundef byval(<3 x i512>) align 256 [[TMP0:%.*]]) #[[ATTR0]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca <3 x i512>, align 256
 // CHECK-NEXT:    [[LOADVECN:%.*]] = load <4 x i512>, ptr [[TMP0]], align 256
@@ -64,7 +64,13 @@ i32x3 v2(i32x3 a) {
 // CHECK-NEXT:    [[LOADVECN3:%.*]] = load <4 x i512>, ptr [[A_ADDR]], align 256
 // CHECK-NEXT:    [[EXTRACTVEC4:%.*]] = shufflevector <4 x i512> [[LOADVECN3]], <4 x i512> poison, <3 x i32> <i32 0, i32 1, i32 2>
 // CHECK-NEXT:    [[ADD:%.*]] = add <3 x i512> [[EXTRACTVEC2]], [[EXTRACTVEC4]]
-// CHECK-NEXT:    ret <3 x i512> [[ADD]]
+// CHECK-NEXT:    [[EXTRACTVEC5:%.*]] = shufflevector <3 x i512> [[ADD]], <3 x i512> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+// CHECK-NEXT:    store <4 x i512> [[EXTRACTVEC5]], ptr [[AGG_RESULT]], align 256
+// CHECK-NEXT:    [[LOADVECN6:%.*]] = load <4 x i512>, ptr [[AGG_RESULT]], align 256
+// CHECK-NEXT:    [[EXTRACTVEC7:%.*]] = shufflevector <4 x i512> [[LOADVECN6]], <4 x i512> poison, <3 x i32> <i32 0, i32 1, i32 2>
+// CHECK-NEXT:    [[EXTRACTVEC8:%.*]] = shufflevector <3 x i512> [[EXTRACTVEC7]], <3 x i512> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+// CHECK-NEXT:    store <4 x i512> [[EXTRACTVEC8]], ptr [[AGG_RESULT]], align 256
+// CHECK-NEXT:    ret void
 //
 i512x3 v3(i512x3 a) {
   return a + a;
@@ -97,7 +103,7 @@ i4x3 v4(i4x3 a) {
 }
 
 // CHECK-LABEL: define dso_local noundef <9 x i8> @_Z2m1u11matrix_typeILm3ELm3EDB8_E(
-// CHECK-SAME: <9 x i8> noundef [[A:%.*]]) #[[ATTR4:[0-9]+]] {
+// CHECK-SAME: <9 x i8> noundef [[A:%.*]]) #[[ATTR3:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [9 x i8], align 1
 // CHECK-NEXT:    store <9 x i8> [[A]], ptr [[A_ADDR]], align 1
@@ -111,7 +117,7 @@ i8x3x3 m1(i8x3x3 a) {
 }
 
 // CHECK-LABEL: define dso_local noundef <9 x i32> @_Z2m2u11matrix_typeILm3ELm3EDB32_E(
-// CHECK-SAME: <9 x i32> noundef [[A:%.*]]) #[[ATTR5:[0-9]+]] {
+// CHECK-SAME: <9 x i32> noundef [[A:%.*]]) #[[ATTR4:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [9 x i32], align 4
 // CHECK-NEXT:    store <9 x i32> [[A]], ptr [[A_ADDR]], align 4
@@ -125,7 +131,7 @@ i32x3x3 m2(i32x3x3 a) {
 }
 
 // CHECK-LABEL: define dso_local noundef <9 x i512> @_Z2m3u11matrix_typeILm3ELm3EDB512_E(
-// CHECK-SAME: <9 x i512> noundef [[A:%.*]]) #[[ATTR6:[0-9]+]] {
+// CHECK-SAME: <9 x i512> noundef [[A:%.*]]) #[[ATTR5:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [9 x i512], align 8
 // CHECK-NEXT:    store <9 x i512> [[A]], ptr [[A_ADDR]], align 8
@@ -139,7 +145,7 @@ i512x3x3 m3(i512x3x3 a) {
 }
 
 // CHECK-LABEL: define dso_local noundef <9 x i4> @_Z2m4u11matrix_typeILm3ELm3EDB4_E(
-// CHECK-SAME: <9 x i4> noundef [[A:%.*]]) #[[ATTR7:[0-9]+]] {
+// CHECK-SAME: <9 x i4> noundef [[A:%.*]]) #[[ATTR6:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[A_ADDR:%.*]] = alloca [9 x i4], align 1
 // CHECK-NEXT:    store <9 x i4> [[A]], ptr [[A_ADDR]], align 1
diff --git a/clang/test/CodeGenCXX/x86_64-empty-base-vector-abi.cpp b/clang/test/CodeGenCXX/x86_64-empty-base-vector-abi.cpp
new file mode 100644
index 00000000000000..4f2ae00883461c
--- /dev/null
+++ b/clang/test/CodeGenCXX/x86_64-empty-base-vector-abi.cpp
@@ -0,0 +1,51 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++20 -emit-llvm -o - %s -target-feature +avx | FileCheck %s --check-prefix=SYSV
+// RUN: %clang_cc1 -triple x86_64-sie-ps5 -std=c++20 -emit-llvm -o - %s -target-feature +avx | FileCheck %s --check-prefix=PS
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++20 -fclang-abi-compat=22 -emit-llvm -o - %s -target-feature +avx | FileCheck %s --check-prefix=CLANG22
+
+typedef unsigned long long v4ull __attribute__((vector_size(32)));
+
+struct EmptyBase {};
+
+struct EmptyBaseThenVector : EmptyBase {
+  v4ull Data;
+};
+
+EmptyBaseThenVector return_empty_base_then_vector() {
+  return {};
+}
+
+unsigned long long pass_empty_base_then_vector(EmptyBaseThenVector X) {
+  return X.Data[0];
+}
+
+struct EmptyField {};
+
+struct EmptyFieldThenVector {
+  [[no_unique_address]] EmptyField E;
+  v4ull Data;
+};
+
+EmptyFieldThenVector return_empty_field_then_vector() {
+  return {};
+}
+
+unsigned long long pass_empty_field_then_vector(EmptyFieldThenVector X) {
+  return X.Data[0];
+}
+
+// Empty base classes are covered by CodeGen/X86/avx-cxx-record.cpp. This test
+// covers [[no_unique_address]] empty fields, which are not handled by the
+// legacy classifier.
+// SYSV-LABEL: define dso_local <4 x i64> @_Z30return_empty_field_then_vectorv()
+// SYSV-LABEL: define dso_local noundef i64 @_Z28pass_empty_field_then_vector20EmptyFieldThenVector(<4 x i64> %X.coerce)
+
+// PlayStation keeps the legacy ABI behavior for both cases.
+// PS-LABEL: define dso_local <4 x i64> @_Z29return_empty_base_then_vectorv()
+// PS-LABEL: define dso_local noundef i64 @_Z27pass_empty_base_then_vector19EmptyBaseThenVector(<4 x i64> %X.coerce)
+// PS-LABEL: define dso_local void @_Z30return_empty_field_then_vectorv(ptr dead_on_unwind noalias writable sret(%struct.EmptyFieldThenVector) align 32 %agg.result)
+// PS-LABEL: define dso_local noundef i64 @_Z28pass_empty_field_then_vector20EmptyFieldThenVector(ptr noundef byval(%struct.EmptyFieldThenVector) align 32 %X)
+
+// Clang 22 ABI compatibility mode keeps the legacy ABI behavior for
+// [[no_unique_address]] empty fields.
+// CLANG22-LABEL: define dso_local void @_Z30return_empty_field_then_vectorv(ptr dead_on_unwind noalias writable sret(%struct.EmptyFieldThenVector) align 32 %agg.result)
+// CLANG22-LABEL: define dso_local noundef i64 @_Z28pass_empty_field_then_vector20EmptyFieldThenVector(ptr noundef byval(%struct.EmptyFieldThenVector) align 32 %X)

>From b97b6e786d8887ce370095b3284f916676a44a1a Mon Sep 17 00:00:00 2001
From: Benjamin Luke <benjamin.luke at sony.com>
Date: Wed, 5 Aug 2026 13:45:57 -0700
Subject: [PATCH 02/18] Update tests to target clang23 compatibility instead of
 clang22

---
 clang/test/CodeGen/X86/x86-illegal-vector.c            | 6 +++---
 clang/test/CodeGen/X86/x86-small-fp-vector-abi.c       | 2 +-
 clang/test/CodeGen/X86/x86_64-atomic-vector-abi.c      | 2 +-
 clang/test/CodeGenCXX/x86_64-empty-base-vector-abi.cpp | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/clang/test/CodeGen/X86/x86-illegal-vector.c b/clang/test/CodeGen/X86/x86-illegal-vector.c
index abb863f28d9451..90e651142b3508 100644
--- a/clang/test/CodeGen/X86/x86-illegal-vector.c
+++ b/clang/test/CodeGen/X86/x86-illegal-vector.c
@@ -1,11 +1,11 @@
 // RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +sse2 -emit-llvm -o - | FileCheck %s --check-prefixes=REGRET128,MEMRET256,MEMRET512
-// RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +sse2 -emit-llvm -o - -fclang-abi-compat=22 | FileCheck %s --check-prefixes=REGRET128,REGRET256,REGRET512
+// RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +sse2 -emit-llvm -o - -fclang-abi-compat=23 | FileCheck %s --check-prefixes=REGRET128,REGRET256,REGRET512
 
 // RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +avx -emit-llvm -o - | FileCheck %s --check-prefixes=REGRET128,REGRET256,MEMRET512
-// RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +avx -emit-llvm -o - -fclang-abi-compat=22 | FileCheck %s --check-prefixes=REGRET128,REGRET256,REGRET512
+// RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +avx -emit-llvm -o - -fclang-abi-compat=23 | FileCheck %s --check-prefixes=REGRET128,REGRET256,REGRET512
 
 // RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +avx512f -emit-llvm -o - | FileCheck %s --check-prefixes=REGRET128,REGRET256,REGRET512
-// RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +avx512f -emit-llvm -o - -fclang-abi-compat=22 | FileCheck %s --check-prefixes=REGRET128,REGRET256,REGRET512
+// RUN: %clang_cc1 -triple x86_64-linux-gnu %s -target-feature +avx512f -emit-llvm -o - -fclang-abi-compat=23 | FileCheck %s --check-prefixes=REGRET128,REGRET256,REGRET512
 
 #define __MM_MALLOC_H
 #include <x86intrin.h>
diff --git a/clang/test/CodeGen/X86/x86-small-fp-vector-abi.c b/clang/test/CodeGen/X86/x86-small-fp-vector-abi.c
index 66f0fec6ee247f..5bc3d577f3d634 100644
--- a/clang/test/CodeGen/X86/x86-small-fp-vector-abi.c
+++ b/clang/test/CodeGen/X86/x86-small-fp-vector-abi.c
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -o - %s | FileCheck %s
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -fclang-abi-compat=22 -emit-llvm -o - %s | FileCheck %s --check-prefix=LEGACY
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -fclang-abi-compat=23 -emit-llvm -o - %s | FileCheck %s --check-prefix=LEGACY
 
 typedef _Float16 v1hf __attribute__((vector_size(2)));
 typedef _Float16 v2hf __attribute__((vector_size(4)));
diff --git a/clang/test/CodeGen/X86/x86_64-atomic-vector-abi.c b/clang/test/CodeGen/X86/x86_64-atomic-vector-abi.c
index 6d0bf403879fb3..518fc434f1e719 100644
--- a/clang/test/CodeGen/X86/x86_64-atomic-vector-abi.c
+++ b/clang/test/CodeGen/X86/x86_64-atomic-vector-abi.c
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -triple x86_64-linux-gnu %s -emit-llvm -o - | FileCheck %s
 // RUN: %clang_cc1 -triple x86_64-linux-gnu -target-feature +avx %s -emit-llvm -o - | FileCheck %s --check-prefix=AVX
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -target-feature +avx -fclang-abi-compat=22 %s -emit-llvm -o - | FileCheck %s --check-prefix=LEGACY
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -target-feature +avx -fclang-abi-compat=23 %s -emit-llvm -o - | FileCheck %s --check-prefix=LEGACY
 
 struct F3 {
   float a[3];
diff --git a/clang/test/CodeGenCXX/x86_64-empty-base-vector-abi.cpp b/clang/test/CodeGenCXX/x86_64-empty-base-vector-abi.cpp
index 4f2ae00883461c..235a8f11d31e75 100644
--- a/clang/test/CodeGenCXX/x86_64-empty-base-vector-abi.cpp
+++ b/clang/test/CodeGenCXX/x86_64-empty-base-vector-abi.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++20 -emit-llvm -o - %s -target-feature +avx | FileCheck %s --check-prefix=SYSV
 // RUN: %clang_cc1 -triple x86_64-sie-ps5 -std=c++20 -emit-llvm -o - %s -target-feature +avx | FileCheck %s --check-prefix=PS
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++20 -fclang-abi-compat=22 -emit-llvm -o - %s -target-feature +avx | FileCheck %s --check-prefix=CLANG22
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++20 -fclang-abi-compat=23 -emit-llvm -o - %s -target-feature +avx | FileCheck %s --check-prefix=CLANG22
 
 typedef unsigned long long v4ull __attribute__((vector_size(32)));
 

>From 5a7c8014215d1a136e5ac5cc852e56974dc3c449 Mon Sep 17 00:00:00 2001
From: Benjamin Luke <benjamin.luke at sony.com>
Date: Wed, 5 Aug 2026 13:46:09 -0700
Subject: [PATCH 03/18] remove un-needed header includes

---
 clang/lib/CodeGen/Targets/X86.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index 64906ea4afcf92..bd09ac27b63764 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -13,8 +13,6 @@
 #include "clang/Basic/SourceLocation.h"
 #include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/Support/ErrorHandling.h"
-#include <cstdint>
 
 using namespace clang;
 using namespace clang::CodeGen;

>From 430285f72d7333c08d56e2edc336045d286fefed Mon Sep 17 00:00:00 2001
From: Benjamin Luke <benjamin.luke at sony.com>
Date: Wed, 5 Aug 2026 13:46:59 -0700
Subject: [PATCH 04/18] Fix test to expect illegal vectors return in mem

---
 clang/test/CodeGen/target-avx-function-abi.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/clang/test/CodeGen/target-avx-function-abi.c b/clang/test/CodeGen/target-avx-function-abi.c
index 6ea0fd39a29f1c..12e1bec8900fc1 100644
--- a/clang/test/CodeGen/target-avx-function-abi.c
+++ b/clang/test/CodeGen/target-avx-function-abi.c
@@ -24,10 +24,12 @@ __attribute__((target("avx512f"))) v16f call_ptr_l512(v16f x) {
   return fp(x);
 }
 
-// SYSV-LABEL: define dso_local <8 x float> @g256(
+// SYSV-LABEL: define dso_local void @g256(
+// SYSV: sret(<8 x float>) align 32
 // SYSV: byval(<8 x float>) align 32
 
-// SYSV-LABEL: define dso_local <16 x float> @g512(
+// SYSV-LABEL: define dso_local void @g512(
+// SYSV: sret(<16 x float>) align 64
 // SYSV: byval(<16 x float>) align 64
 
 // SYSV-LABEL: define dso_local <8 x float> @l256(<8 x float> noundef %x)

>From 25f5e8be59fd39ba437352a9e33695f776486c6e Mon Sep 17 00:00:00 2001
From: Benjamin Luke <benjamin.luke at sony.com>
Date: Fri, 4 Sep 2026 10:44:57 -0700
Subject: [PATCH 05/18] clarify comment about int128 vectors

---
 clang/lib/CodeGen/Targets/X86.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index bd09ac27b63764..63bed0c04caa2a 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -2204,10 +2204,10 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
       return;
     }
 
-    // Vectors of __int128 wider than 128 bits classify as MEMORY: each
-    // __int128 lane contributes two INTEGER eightbytes, and post-merge cleanup
-    // only keeps wider objects in registers when they are SSE followed by
-    // SSEUp.
+    // The psABI does not specify how arbitrary GNU vector extension types such
+    // as vectors of __int128 are classified. Match GCC, which passes vectors
+    // of __int128 wider than 128 bits in memory on platforms that opt in to
+    // this compatibility behavior.
     if (passInt128VectorsInMem() && Size > 128 &&
         (ElementType->isSpecificBuiltinType(BuiltinType::Int128) ||
          ElementType->isSpecificBuiltinType(BuiltinType::UInt128))) {

>From 971b1c7b05e4d6aee7fa1e2362c446a92452b35e Mon Sep 17 00:00:00 2001
From: Benjamin Luke <benjamin.luke at sony.com>
Date: Fri, 4 Sep 2026 10:58:27 -0700
Subject: [PATCH 06/18] Port PR 216777 to new classification algorithm

---
 clang/lib/CodeGen/Targets/X86.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index 63bed0c04caa2a..8892ec626b192c 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -2398,8 +2398,9 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
       uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
       bool BitField = i->isBitField();
 
-      // Ignore padding bit-fields.
-      if (BitField && i->isUnnamedBitField())
+      // Ignore zero-length bit-fields. Other unnamed bit-fields are real
+      // storage and classify like named ones, matching GCC.
+      if (BitField && i->isZeroLengthBitField())
         continue;
 
       bool IsInMemory =
@@ -2421,7 +2422,7 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
       // structure to be passed in memory even if unaligned, and
       // therefore they can straddle an eightbyte.
       if (BitField) {
-        assert(!i->isUnnamedBitField());
+        assert(!i->isZeroLengthBitField());
         uint64_t BitSize = i->getBitWidthValue();
         for (uint64_t BitOffset = Offset, End = Offset + BitSize;
              BitOffset < End;

>From 42e8f51cbf053f284f03f29c0cd673fd0d9b98f8 Mon Sep 17 00:00:00 2001
From: Benjamin Luke <benjamin.luke at sony.com>
Date: Sat, 12 Sep 2026 21:16:11 -0700
Subject: [PATCH 07/18] fixup tests after rebase

---
 clang/test/CodeGen/X86/x86_64-atomic-abi.c | 23 ++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/clang/test/CodeGen/X86/x86_64-atomic-abi.c b/clang/test/CodeGen/X86/x86_64-atomic-abi.c
index c5e20e9c5ff338..ada36a7cc7cdc5 100644
--- a/clang/test/CodeGen/X86/x86_64-atomic-abi.c
+++ b/clang/test/CodeGen/X86/x86_64-atomic-abi.c
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CLANG24
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fclang-abi-compat=23 %s -o - | FileCheck %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fexperimental-abi-lowering %s -o - | FileCheck %s
 
 struct Floats {
@@ -19,6 +20,8 @@ void take_floats(struct Floats s) {}
 
 // CHECK-LABEL: define dso_local void @take_floats(
 // CHECK-SAME: <2 x float> %{{.*}})
+// CLANG24-LABEL: define dso_local void @take_floats(
+// CLANG24-SAME: <2 x float> %{{.*}})
 
 // A scalar atomic still has the value type's evaluation kind, so it is a
 // direct float argument.
@@ -26,27 +29,36 @@ void take_atomic_float(_Atomic(float) value) {}
 
 // CHECK-LABEL: define dso_local void @take_atomic_float(
 // CHECK-SAME: float %{{.*}})
+// CLANG24-LABEL: define dso_local void @take_atomic_float(
+// CLANG24-SAME: float %{{.*}})
 
-// Atomic fields classify Memory rather than inheriting the underlying float's
-// SSE class, so the record is passed on the stack.
+// The experimental classifier classifies atomic fields as Memory, so the
+// record is passed on the stack. The Clang 24 classifier instead classifies
+// each atomic field as its underlying float type, producing an SSE class.
 void take_atomic_floats(struct AtomicFloats s) {}
 
 // CHECK-LABEL: define dso_local void @take_atomic_floats(
 // CHECK-SAME: ptr noundef byval(%struct.AtomicFloats) align 8 %{{.*}})
+// CLANG24-LABEL: define dso_local void @take_atomic_floats(
+// CLANG24-SAME: <2 x float> %{{.*}})
 
 // The same classification applies when that record is nested.
 void take_nested_atomic_floats(struct NestedAtomicFloats s) {}
 
 // CHECK-LABEL: define dso_local void @take_nested_atomic_floats(
 // CHECK-SAME: ptr noundef byval(%struct.NestedAtomicFloats) align 8 %{{.*}})
+// CLANG24-LABEL: define dso_local void @take_nested_atomic_floats(
+// CLANG24-SAME: <2 x float> %{{.*}})
 
 // Atomic types preserve any size inflation relative to their value type.
 void take_padded_atomic(_Atomic(ThreeBytes) value) {}
 
 // CHECK-LABEL: define dso_local void @take_padded_atomic(
 // CHECK-SAME: ptr noundef byval({ %struct.ThreeBytes, [1 x i8] }) align 8
+// CLANG24-LABEL: define dso_local void @take_padded_atomic(
+// CLANG24-SAME: ptr noundef byval({ %struct.ThreeBytes, [1 x i8] }) align 8
 
-// Aggregate returns use the same Memory classification.
+// Return classification follows the same distinction between classifiers.
 struct AtomicFloats return_atomic_floats(void);
 void call_return_atomic_floats(void) { return_atomic_floats(); }
 
@@ -55,3 +67,6 @@ void call_return_atomic_floats(void) { return_atomic_floats(); }
 // CHECK-SAME: ptr dead_on_unwind writable sret(%struct.AtomicFloats) align 4
 // CHECK: declare void @return_atomic_floats(
 // CHECK-SAME: ptr dead_on_unwind writable sret(%struct.AtomicFloats) align 4)
+// CLANG24-LABEL: define dso_local void @call_return_atomic_floats(
+// CLANG24: call <2 x float> @return_atomic_floats()
+// CLANG24: declare <2 x float> @return_atomic_floats()

>From acff2ec8659c761c04e940ef8c6e39800be68256 Mon Sep 17 00:00:00 2001
From: Benjamin Luke <benjamin.luke at sony.com>
Date: Sat, 12 Sep 2026 21:16:41 -0700
Subject: [PATCH 08/18] Fix bug with enum types dropping their classification
 on the floor for the legacy classifier

---
 clang/lib/CodeGen/Targets/X86.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index 8892ec626b192c..23d74d67deccfb 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -2527,8 +2527,7 @@ void X86_64ABIInfo::classifyClang23(QualType Ty, uint64_t OffsetBase,
 
   if (const auto *ED = Ty->getAsEnumDecl()) {
     // Classify the underlying integer type.
-    SmallVector<Class> IntEightBytes = {Lo, Hi};
-    classifyClang23(ED->getIntegerType(), OffsetBase, IntEightBytes, isNamedArg,
+    classifyClang23(ED->getIntegerType(), OffsetBase, EightBytes, isNamedArg,
                     IsRegCall);
     return;
   }

>From f66599ec9fdaa384bd3c41d192723aa06f4b8236 Mon Sep 17 00:00:00 2001
From: Benjamin Luke <benjamin.luke at sony.com>
Date: Sun, 13 Sep 2026 07:26:08 -0700
Subject: [PATCH 09/18] WIP on factoring out common logic between clang23 and
 clang24 classifiers

---
 clang/lib/CodeGen/Targets/X86.cpp | 420 ++++++++++++++----------------
 1 file changed, 193 insertions(+), 227 deletions(-)

diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index 23d74d67deccfb..c7cdfbb6f951bc 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -1324,6 +1324,18 @@ class X86_64ABIInfo : public ABIInfo {
     Memory
   };
 
+  struct ClassPair {
+    Class Lo = NoClass;
+    Class Hi = NoClass;
+  };
+
+  struct VectorTypeInfo {
+    QualType ElementType;
+    bool IsSingleElement;
+    bool IsFPElement;
+    bool IsInt128Element;
+  };
+
   /// merge - Implement the X86_64 ABI merging algorithm.
   ///
   /// Merge an accumulating classification \arg Accum with a field
@@ -1335,32 +1347,45 @@ class X86_64ABIInfo : public ABIInfo {
   /// should just return Memory for the aggregate).
   static Class merge(Class Accum, Class Field);
 
-  /// classify - Determine the x86_64 register classes in which the
-  /// given type T should be passed.
-  ///
-  /// \param Lo - The classification for the parts of the type
-  /// residing in the low word of the containing object.
-  ///
-  /// \param Hi - The classification for the parts of the type
-  /// residing in the high word of the containing object.
-  ///
-  /// \param OffsetBase - The bit offset of this type in the
-  /// containing object.  Some parameters are classified different
-  /// depending on whether they straddle an eightbyte boundary.
-  ///
-  /// \param isNamedArg - Whether the argument in question is a "named"
-  /// argument, as used in AMD64-ABI 3.5.7.
-  ///
-  /// \param IsRegCall - Whether the calling conversion is regcall.
-  ///
-  /// If a word is unused its result will be NoClass; if a type should
-  /// be passed in Memory then at least the classification of \arg Lo
-  /// will be Memory.
+  /// Return the classes intrinsic to the builtin type \param BT. The caller is
+  /// responsible for placing them at the type's offset.
+  ClassPair getBuiltinTypeClassification(const BuiltinType *BT) const;
+
+  /// Return the classes intrinsic to the complex type \param CT. The caller is
+  /// responsible for boundary splitting and class placement.
+  ClassPair getComplexTypeClassification(const ComplexType *CT) const;
+
+  /// Return the classes intrinsic to the bit-precise integer type \param BT.
+  /// The caller is responsible for class placement.
+  static ClassPair getBitIntTypeClassification(const BitIntType *BT);
+
+  /// Collect the canonical element type and shared classification properties
+  /// of the vector \param VT or matrix \param MT. At least one must be
+  /// non-null.
+  VectorTypeInfo getVectorTypeInfo(const VectorType *VT,
+                                   const MatrixType *MT = nullptr) const;
+
+  /// Return true when \param RT must be passed in memory without inspecting its
+  /// bases or fields because of its size, record ABI, or flexible array member.
+  bool isRecordPassedInMemory(const RecordType *RT, uint64_t Size) const;
+
+  /// Determine the x86_64 register classes used to pass or return \p T.
   ///
-  /// The \arg Lo class will be NoClass iff the argument is ignored.
+  /// Each element of \p EightBytes describes an eightbyte in the containing
+  /// object. The Clang 24 classifier can produce an entry for every eightbyte,
+  /// while the legacy classifier represents the result using its first two
+  /// entries. NoClass denotes an unused eightbyte and Memory denotes a value
+  /// that must be passed in memory.
   ///
-  /// If the \arg Lo class is ComplexX87, then the \arg Hi class will
-  /// also be ComplexX87.
+  /// \param T The type to classify.
+  /// \param OffsetBase The bit offset of \p T in its containing object.
+  /// Classification can depend on whether the type straddles an eightbyte
+  /// boundary.
+  /// \param EightBytes The resulting eightbyte classes.
+  /// \param isNamedArg Whether the argument is named rather than part of a
+  /// variadic argument list, as used in AMD64-ABI 3.5.7.
+  /// \param isRegCall Whether the calling convention is regcall. Regcall uses
+  /// the legacy classifier.
   void classify(QualType T, uint64_t OffsetBase,
                 SmallVectorImpl<Class> &EightBytes, bool isNamedArg,
                 bool isRegCall = false) const;
@@ -1966,6 +1991,108 @@ X86_64ABIInfo::Class X86_64ABIInfo::merge(Class Accum, Class Field) {
   return SSE;
 }
 
+X86_64ABIInfo::ClassPair
+X86_64ABIInfo::getBuiltinTypeClassification(const BuiltinType *BT) const {
+  BuiltinType::Kind K = BT->getKind();
+
+  if (K == BuiltinType::Void)
+    return {};
+  if (K == BuiltinType::Int128 || K == BuiltinType::UInt128)
+    return {Integer, Integer};
+  if (K >= BuiltinType::Bool && K <= BuiltinType::LongLong)
+    return {Integer};
+  if (K == BuiltinType::Float || K == BuiltinType::Double ||
+      K == BuiltinType::Float16 || K == BuiltinType::BFloat16)
+    return {SSE};
+  if (K == BuiltinType::Float128)
+    return {SSE, SSEUp};
+  if (K == BuiltinType::LongDouble) {
+    const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
+    if (LDF == &llvm::APFloat::IEEEquad())
+      return {SSE, SSEUp};
+    if (LDF == &llvm::APFloat::x87DoubleExtended())
+      return {X87, X87Up};
+    if (LDF == &llvm::APFloat::IEEEdouble())
+      return {SSE};
+    llvm_unreachable("unexpected long double representation!");
+  }
+
+  return {Memory};
+}
+
+X86_64ABIInfo::ClassPair
+X86_64ABIInfo::getComplexTypeClassification(const ComplexType *CT) const {
+  QualType ET = getContext().getCanonicalType(CT->getElementType());
+  uint64_t Size = getContext().getTypeSize(QualType(CT, 0));
+
+  if (ET->isIntegralOrEnumerationType()) {
+    if (Size <= 64)
+      return {Integer};
+    if (Size <= 128)
+      return {Integer, Integer};
+    llvm_unreachable("unexpected complex integer type size");
+  }
+  if (ET->isFloat16Type() || ET == getContext().FloatTy || ET->isBFloat16Type())
+    return {SSE};
+  if (ET == getContext().DoubleTy)
+    return {SSE, SSE};
+  if (ET == getContext().LongDoubleTy) {
+    const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
+    if (LDF == &llvm::APFloat::IEEEquad())
+      return {Memory};
+    if (LDF == &llvm::APFloat::x87DoubleExtended())
+      return {ComplexX87};
+    if (LDF == &llvm::APFloat::IEEEdouble())
+      return {SSE, SSE};
+    llvm_unreachable("unexpected long double representation!");
+  }
+  if (ET->isFloat128Type())
+    return {Memory};
+  llvm_unreachable("unexpected complex element type");
+}
+
+X86_64ABIInfo::ClassPair
+X86_64ABIInfo::getBitIntTypeClassification(const BitIntType *BT) {
+  if (BT->getNumBits() <= 64)
+    return {Integer};
+  if (BT->getNumBits() <= 128)
+    return {Integer, Integer};
+  return {Memory};
+}
+
+X86_64ABIInfo::VectorTypeInfo
+X86_64ABIInfo::getVectorTypeInfo(const VectorType *VT,
+                                 const MatrixType *MT) const {
+  assert((VT || MT) && "expected a vector or matrix type");
+  QualType ElementType = getContext().getCanonicalType(
+      VT ? VT->getElementType() : MT->getElementType());
+  bool IsFPElement = ElementType->isFloat16Type() ||
+                     ElementType->isBFloat16Type() ||
+                     ElementType == getContext().FloatTy ||
+                     ElementType == getContext().DoubleTy;
+  bool IsInt128Element =
+      ElementType->isSpecificBuiltinType(BuiltinType::Int128) ||
+      ElementType->isSpecificBuiltinType(BuiltinType::UInt128);
+  return {ElementType, VT && VT->getNumElements() == 1, IsFPElement,
+          IsInt128Element};
+}
+
+bool X86_64ABIInfo::isRecordPassedInMemory(const RecordType *RT,
+                                           uint64_t Size) const {
+  // AMD64-ABI 3.2.3p2: Rule 1. Objects larger than eight eightbytes have class
+  // MEMORY.
+  if (Size > 512)
+    return true;
+
+  // Rule 2. C++ objects with a non-trivial copy constructor or destructor are
+  // passed by invisible reference.
+  if (getRecordArgABI(RT, getCXXABI()))
+    return true;
+
+  // Assume variable sized types are passed in memory.
+  return RT->getDecl()->getDefinitionOrSelf()->hasFlexibleArrayMember();
+}
+
 void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
                              SmallVectorImpl<Class> &EightBytes,
                              bool isNamedArg, bool IsRegCall) const {
@@ -2053,6 +2180,17 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
     SetEightByte(llvm::alignDown(Offset, uint64_t(64)) + 64, HiClass);
   };
 
+  auto SetClassPair = [&](uint64_t Offset, ClassPair Classes) {
+    if (Classes.Lo == Memory || Classes.Hi == Memory) {
+      ClassifyAsMemory();
+      return;
+    }
+    if (Classes.Lo != NoClass)
+      SetEightByte(Offset, Classes.Lo);
+    if (Classes.Hi != NoClass)
+      SetEightByte(llvm::alignDown(Offset, uint64_t(64)) + 64, Classes.Hi);
+  };
+
   auto SetTouchedEightBytes = [&](uint64_t Size, Class C) {
     for (uint64_t Offset = OffsetBase, End = OffsetBase + Size; Offset < End;
          Offset = llvm::alignDown(Offset, uint64_t(64)) + 64)
@@ -2097,40 +2235,7 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
 
   // Built In Types
   if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
-    BuiltinType::Kind K = BT->getKind();
-
-    if (K == BuiltinType::Void)
-      return;
-    if (K == BuiltinType::Int128 || K == BuiltinType::UInt128) {
-      SetAdjacentEightBytes(OffsetBase, Integer, Integer);
-      return;
-    }
-    if (K >= BuiltinType::Bool && K <= BuiltinType::LongLong) {
-      SetEightByte(OffsetBase, Integer);
-      return;
-    }
-    if (K == BuiltinType::Float || K == BuiltinType::Double ||
-        K == BuiltinType::Float16 || K == BuiltinType::BFloat16) {
-      SetEightByte(OffsetBase, SSE);
-      return;
-    }
-    if (K == BuiltinType::Float128) {
-      SetAdjacentEightBytes(OffsetBase, SSE, SSEUp);
-      return;
-    }
-    if (K == BuiltinType::LongDouble) {
-      const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
-      if (LDF == &llvm::APFloat::IEEEquad())
-        SetAdjacentEightBytes(OffsetBase, SSE, SSEUp);
-      else if (LDF == &llvm::APFloat::x87DoubleExtended())
-        SetAdjacentEightBytes(OffsetBase, X87, X87Up);
-      else if (LDF == &llvm::APFloat::IEEEdouble())
-        SetEightByte(OffsetBase, SSE);
-      else
-        llvm_unreachable("unexpected long double representation!");
-      return;
-    }
-    ClassifyAsMemory();
+    SetClassPair(OffsetBase, getBuiltinTypeClassification(BT));
     return;
   }
 
@@ -2159,25 +2264,8 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
   // same element type.
   const VectorType *VT = Ty->getAs<VectorType>();
   const MatrixType *MT = Ty->getAs<MatrixType>();
-  bool IsVectorOrMatrix = VT != nullptr || MT != nullptr;
-  if (IsVectorOrMatrix) {
-    QualType ElementType;
-    bool IsSingleElementVector = false;
-    if (VT) {
-      ElementType = VT->getElementType();
-      IsSingleElementVector = VT->getNumElements() == 1;
-    } else if (MT) {
-      ElementType = MT->getElementType();
-    } else {
-      llvm_unreachable("invalid vector or matrix type");
-    }
-    ElementType = getContext().getCanonicalType(ElementType);
-
-    auto IsFPVectorElement = [&]() {
-      return ElementType->isFloat16Type() || ElementType->isBFloat16Type() ||
-             ElementType == getContext().FloatTy ||
-             ElementType == getContext().DoubleTy;
-    };
+  if (VT || MT) {
+    VectorTypeInfo Info = getVectorTypeInfo(VT, MT);
 
     auto SetEightByteClasses = [&](Class FirstClass, Class RestClass) {
       // Vector objects should not start partway through one eightbyte and
@@ -2197,7 +2285,7 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
         SetEightByte(Offset, RestClass);
     };
 
-    if (IsSingleElementVector && IsFPVectorElement()) {
+    if (Info.IsSingleElement && Info.IsFPElement) {
       // GCC passes single-element floating-point vectors in memory. This is a
       // compatibility rule, not a psABI classification rule.
       ClassifyAsMemory();
@@ -2208,9 +2296,7 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
     // as vectors of __int128 are classified. Match GCC, which passes vectors
     // of __int128 wider than 128 bits in memory on platforms that opt in to
     // this compatibility behavior.
-    if (passInt128VectorsInMem() && Size > 128 &&
-        (ElementType->isSpecificBuiltinType(BuiltinType::Int128) ||
-         ElementType->isSpecificBuiltinType(BuiltinType::UInt128))) {
+    if (passInt128VectorsInMem() && Size > 128 && Info.IsInt128Element) {
       ClassifyAsMemory();
       return;
     }
@@ -2226,7 +2312,7 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
       // 4 bytes - <4 x char>, <2 x short>, <1 x int>
       // 2 bytes - <2 x char>, <1 x short>
       // 1 byte  - <1 x char>
-      Class C = IsFPVectorElement() ? SSE : Integer;
+      Class C = Info.IsFPElement ? SSE : Integer;
 
       SetEightByteClasses(C, C);
     } else if (Size == 64) {
@@ -2245,35 +2331,11 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
 
   // Complex types
   if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
-    QualType ET = getContext().getCanonicalType(CT->getElementType());
-
-    uint64_t Size = getContext().getTypeSize(Ty);
-    if (ET->isIntegralOrEnumerationType()) {
-      if (Size <= 64)
-        SetEightByte(OffsetBase, Integer);
-      else if (Size <= 128)
-        SetAdjacentEightBytes(OffsetBase, Integer, Integer);
-    } else if (ET->isFloat16Type() || ET == getContext().FloatTy ||
-               ET->isBFloat16Type()) {
-      SetEightByte(OffsetBase, SSE);
-    } else if (ET == getContext().DoubleTy) {
-      SetAdjacentEightBytes(OffsetBase, SSE, SSE);
-    } else if (ET == getContext().LongDoubleTy) {
-      const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
-      if (LDF == &llvm::APFloat::IEEEquad())
-        ClassifyAsMemory();
-      else if (LDF == &llvm::APFloat::x87DoubleExtended())
-        SetEightByte(OffsetBase, ComplexX87);
-      else if (LDF == &llvm::APFloat::IEEEdouble())
-        SetAdjacentEightBytes(OffsetBase, SSE, SSE);
-      else
-        llvm_unreachable("unexpected long double representation!");
-    } else if (ET->isFloat128Type()) {
-      ClassifyAsMemory();
-    }
+    SetClassPair(OffsetBase, getComplexTypeClassification(CT));
 
     // If this complex type crosses an eightbyte boundary then it
     // should be split.
+    QualType ET = getContext().getCanonicalType(CT->getElementType());
     uint64_t EB_Real = (OffsetBase) / 64;
     uint64_t EB_Imag = (OffsetBase + getContext().getTypeSize(ET)) / 64;
     if (EightBytes[EB_Imag] == NoClass && EB_Real != EB_Imag)
@@ -2285,15 +2347,7 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
 
   // BitInt Types
   if (const auto *EITy = Ty->getAs<BitIntType>()) {
-    if (EITy->getNumBits() <= 64)
-      SetEightByte(OffsetBase, Integer);
-    else if (EITy->getNumBits() <= 128)
-      SetAdjacentEightBytes(OffsetBase, Integer, Integer);
-    else {
-      // Larger values need to get passed in memory.
-      ClassifyAsMemory();
-    }
-
+    SetClassPair(OffsetBase, getBitIntTypeClassification(EITy));
     return;
   }
 
@@ -2338,31 +2392,12 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
 
   // CXX Record Type
   if (const RecordType *RT = Ty->getAsCanonical<RecordType>()) {
-    // uint64_t Size = getContext().getTypeSize(Ty);
-
-    // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
-    // than eight eightbytes, ..., it has class MEMORY.
-    if (Size > 512) {
-      ClassifyAsMemory();
-      return;
-    }
-
-    // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial
-    // copy constructor or a non-trivial destructor, it is passed by
-    // invisible reference.
-    if (getRecordArgABI(RT, getCXXABI())) {
+    if (isRecordPassedInMemory(RT, Size)) {
       ClassifyAsMemory();
       return;
     }
 
     const RecordDecl *RD = RT->getDecl()->getDefinitionOrSelf();
-
-    // Assume variable sized types are passed in memory.
-    if (RD->hasFlexibleArrayMember()) {
-      ClassifyAsMemory();
-      return;
-    }
-
     const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
 
     // If this is a C++ record, classify the bases first.
@@ -2445,10 +2480,6 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
 void X86_64ABIInfo::classifyClang23(QualType Ty, uint64_t OffsetBase,
                                     SmallVectorImpl<Class> &EightBytes,
                                     bool isNamedArg, bool IsRegCall) const {
-  // FIXME: This code can be simplified by introducing a simple value class
-  // for Class pairs with appropriate constructor methods for the various
-  // situations.
-
   // FIXME: Some of the split computations are wrong; unaligned vectors
   // shouldn't be passed in registers for example, so there is no chance they
   // can straddle an eightbyte. Verify & simplify.
@@ -2491,37 +2522,17 @@ void X86_64ABIInfo::classifyClang23(QualType Ty, uint64_t OffsetBase,
   Class &Current = OffsetBase < 64 ? Lo : Hi;
   Current = Memory;
 
-  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
-    BuiltinType::Kind k = BT->getKind();
-
-    if (k == BuiltinType::Void) {
-      Current = NoClass;
-    } else if (k == BuiltinType::Int128 || k == BuiltinType::UInt128) {
-      Lo = Integer;
-      Hi = Integer;
-    } else if (k >= BuiltinType::Bool && k <= BuiltinType::LongLong) {
-      Current = Integer;
-    } else if (k == BuiltinType::Float || k == BuiltinType::Double ||
-               k == BuiltinType::Float16 || k == BuiltinType::BFloat16) {
-      Current = SSE;
-    } else if (k == BuiltinType::Float128) {
-      Lo = SSE;
-      Hi = SSEUp;
-    } else if (k == BuiltinType::LongDouble) {
-      const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
-      if (LDF == &llvm::APFloat::IEEEquad()) {
-        Lo = SSE;
-        Hi = SSEUp;
-      } else if (LDF == &llvm::APFloat::x87DoubleExtended()) {
-        Lo = X87;
-        Hi = X87Up;
-      } else if (LDF == &llvm::APFloat::IEEEdouble()) {
-        Current = SSE;
-      } else
-        llvm_unreachable("unexpected long double representation!");
+  auto SetClassPair = [&](ClassPair Classes) {
+    if (Classes.Hi == NoClass) {
+      Current = Classes.Lo;
+      return;
     }
-    // FIXME: _Decimal32 and _Decimal64 are SSE.
-    // FIXME: _float128 and _Decimal128 are (SSE, SSEUp).
+    Lo = Classes.Lo;
+    Hi = Classes.Hi;
+  };
+
+  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
+    SetClassPair(getBuiltinTypeClassification(BT));
     return;
   }
 
@@ -2562,6 +2573,7 @@ void X86_64ABIInfo::classifyClang23(QualType Ty, uint64_t OffsetBase,
 
   if (const VectorType *VT = Ty->getAs<VectorType>()) {
     uint64_t Size = getContext().getTypeSize(VT);
+    VectorTypeInfo Info = getVectorTypeInfo(VT);
     if (Size == 1 || Size == 8 || Size == 16 || Size == 32) {
       // gcc passes the following as integer:
       // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>
@@ -2576,20 +2588,18 @@ void X86_64ABIInfo::classifyClang23(QualType Ty, uint64_t OffsetBase,
       if (EB_Lo != EB_Hi)
         Hi = Lo;
     } else if (Size == 64) {
-      QualType ElementType = VT->getElementType();
-
       // gcc passes <1 x double> in memory. :(
-      if (ElementType->isSpecificBuiltinType(BuiltinType::Double))
+      if (Info.ElementType->isSpecificBuiltinType(BuiltinType::Double))
         return;
 
       // gcc passes <1 x long long> as SSE but clang used to unconditionally
       // pass them as integer.  For platforms where clang is the de facto
       // platform compiler, we must continue to use integer.
       if (!classifyIntegerMMXAsSSE() &&
-          (ElementType->isSpecificBuiltinType(BuiltinType::LongLong) ||
-           ElementType->isSpecificBuiltinType(BuiltinType::ULongLong) ||
-           ElementType->isSpecificBuiltinType(BuiltinType::Long) ||
-           ElementType->isSpecificBuiltinType(BuiltinType::ULong)))
+          (Info.ElementType->isSpecificBuiltinType(BuiltinType::LongLong) ||
+           Info.ElementType->isSpecificBuiltinType(BuiltinType::ULongLong) ||
+           Info.ElementType->isSpecificBuiltinType(BuiltinType::Long) ||
+           Info.ElementType->isSpecificBuiltinType(BuiltinType::ULong)))
         Current = Integer;
       else
         Current = SSE;
@@ -2600,12 +2610,8 @@ void X86_64ABIInfo::classifyClang23(QualType Ty, uint64_t OffsetBase,
         Hi = Lo;
     } else if (Size == 128 ||
                (isNamedArg && Size <= getNativeVectorSizeForAVXABI(AVXLevel))) {
-      QualType ElementType = VT->getElementType();
-
       // gcc passes 256 and 512 bit <X x __int128> vectors in memory. :(
-      if (passInt128VectorsInMem() && Size != 128 &&
-          (ElementType->isSpecificBuiltinType(BuiltinType::Int128) ||
-           ElementType->isSpecificBuiltinType(BuiltinType::UInt128)))
+      if (passInt128VectorsInMem() && Size != 128 && Info.IsInt128Element)
         return;
 
       // Arguments of 256-bits are split into four eightbyte chunks. The
@@ -2629,33 +2635,11 @@ void X86_64ABIInfo::classifyClang23(QualType Ty, uint64_t OffsetBase,
   }
 
   if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
-    QualType ET = getContext().getCanonicalType(CT->getElementType());
-
-    uint64_t Size = getContext().getTypeSize(Ty);
-    if (ET->isIntegralOrEnumerationType()) {
-      if (Size <= 64)
-        Current = Integer;
-      else if (Size <= 128)
-        Lo = Hi = Integer;
-    } else if (ET->isFloat16Type() || ET == getContext().FloatTy ||
-               ET->isBFloat16Type()) {
-      Current = SSE;
-    } else if (ET == getContext().DoubleTy) {
-      Lo = Hi = SSE;
-    } else if (ET == getContext().LongDoubleTy) {
-      const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
-      if (LDF == &llvm::APFloat::IEEEquad())
-        Current = Memory;
-      else if (LDF == &llvm::APFloat::x87DoubleExtended())
-        Current = ComplexX87;
-      else if (LDF == &llvm::APFloat::IEEEdouble())
-        Lo = Hi = SSE;
-      else
-        llvm_unreachable("unexpected long double representation!");
-    }
+    SetClassPair(getComplexTypeClassification(CT));
 
     // If this complex type crosses an eightbyte boundary then it
     // should be split.
+    QualType ET = getContext().getCanonicalType(CT->getElementType());
     uint64_t EB_Real = (OffsetBase) / 64;
     uint64_t EB_Imag = (OffsetBase + getContext().getTypeSize(ET)) / 64;
     if (Hi == NoClass && EB_Real != EB_Imag)
@@ -2665,11 +2649,7 @@ void X86_64ABIInfo::classifyClang23(QualType Ty, uint64_t OffsetBase,
   }
 
   if (const auto *EITy = Ty->getAs<BitIntType>()) {
-    if (EITy->getNumBits() <= 64)
-      Current = Integer;
-    else if (EITy->getNumBits() <= 128)
-      Lo = Hi = Integer;
-    // Larger values need to get passed in memory.
+    SetClassPair(getBitIntTypeClassification(EITy));
     return;
   }
 
@@ -2727,24 +2707,10 @@ void X86_64ABIInfo::classifyClang23(QualType Ty, uint64_t OffsetBase,
 
   if (const RecordType *RT = Ty->getAsCanonical<RecordType>()) {
     uint64_t Size = getContext().getTypeSize(Ty);
-
-    // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
-    // than eight eightbytes, ..., it has class MEMORY.
-    if (Size > 512)
-      return;
-
-    // AMD64-ABI 3.2.3p2: Rule 2. If a C++ object has either a non-trivial
-    // copy constructor or a non-trivial destructor, it is passed by invisible
-    // reference.
-    if (getRecordArgABI(RT, getCXXABI()))
+    if (isRecordPassedInMemory(RT, Size))
       return;
 
     const RecordDecl *RD = RT->getDecl()->getDefinitionOrSelf();
-
-    // Assume variable sized types are passed in memory.
-    if (RD->hasFlexibleArrayMember())
-      return;
-
     const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
 
     // Reset Lo class, this will be recomputed.

>From 78004bf55d88ce0eb38063c2b0d8a197e00b5642 Mon Sep 17 00:00:00 2001
From: Benjamin Luke <benjamin.luke at sony.com>
Date: Sun, 13 Sep 2026 08:16:39 -0700
Subject: [PATCH 10/18] update comment to include open CL pipes

---
 clang/lib/CodeGen/Targets/X86.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index c7cdfbb6f951bc..9eb462f5f68602 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -2246,7 +2246,7 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
     return;
   }
 
-  // Ptrs
+  // Pointers and OpenCL pipes have pointer-sized runtime representations.
   if (Ty->hasPointerRepresentation() || Ty->isPipeType()) {
     SetEightByte(OffsetBase, Integer);
     return;

>From 52f5d4421b01e431c3e9b19bff638e3608c9c223 Mon Sep 17 00:00:00 2001
From: Benjamin Luke <benjamin.luke at sony.com>
Date: Sun, 13 Sep 2026 13:07:45 -0700
Subject: [PATCH 11/18] factored out more common logic between the two
 classification paths

---
 clang/lib/CodeGen/Targets/X86.cpp | 200 +++++++++++-------------------
 1 file changed, 75 insertions(+), 125 deletions(-)

diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index 9eb462f5f68602..9c3e08132593e1 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -1351,20 +1351,31 @@ class X86_64ABIInfo : public ABIInfo {
   /// responsible for placing them at the type's offset.
   ClassPair getBuiltinTypeClassification(const BuiltinType *BT) const;
 
-  /// Return the classes intrinsic to the complex type \param CT. The caller is
-  /// responsible for boundary splitting and class placement.
-  ClassPair getComplexTypeClassification(const ComplexType *CT) const;
+  /// Return the classes intrinsic to the complex type \param CT, including any
+  /// splitting required at \param Offset. The caller is responsible for class
+  /// placement.
+  ClassPair getComplexTypeClassification(const ComplexType *CT,
+                                         uint64_t Offset) const;
 
   /// Return the classes intrinsic to the bit-precise integer type \param BT.
   /// The caller is responsible for class placement.
   static ClassPair getBitIntTypeClassification(const BitIntType *BT);
 
+  /// Return \param C for each eightbyte touched by the bit span beginning at
+  /// \param Offset. The result is relative to the first touched eightbyte.
+  static ClassPair getClassPairForSpan(uint64_t Offset, uint64_t Size, Class C);
+
   /// Collect the canonical element type and shared classification properties
   /// of the vector \param VT or matrix \param MT. At least one must be
   /// non-null.
   VectorTypeInfo getVectorTypeInfo(const VectorType *VT,
                                    const MatrixType *MT = nullptr) const;
 
+  /// Return true when \param AT must be passed in memory because of the SysV
+  /// size limit or the alignment of its base. Regcall ignores the size limit.
+  bool isArrayPassedInMemory(const ConstantArrayType *AT, uint64_t Offset,
+                             uint64_t Size, bool IsRegCall) const;
+
   /// Return true when \param RT must be passed in memory without inspecting its
   /// bases or fields because of its size, record ABI, or flexible array member.
   bool isRecordPassedInMemory(const RecordType *RT, uint64_t Size) const;
@@ -2021,34 +2032,46 @@ X86_64ABIInfo::getBuiltinTypeClassification(const BuiltinType *BT) const {
 }
 
 X86_64ABIInfo::ClassPair
-X86_64ABIInfo::getComplexTypeClassification(const ComplexType *CT) const {
+X86_64ABIInfo::getComplexTypeClassification(const ComplexType *CT,
+                                            uint64_t Offset) const {
   QualType ET = getContext().getCanonicalType(CT->getElementType());
+  uint64_t ElementSize = getContext().getTypeSize(ET);
   uint64_t Size = getContext().getTypeSize(QualType(CT, 0));
+  ClassPair Classes;
 
   if (ET->isIntegralOrEnumerationType()) {
     if (Size <= 64)
-      return {Integer};
-    if (Size <= 128)
-      return {Integer, Integer};
-    llvm_unreachable("unexpected complex integer type size");
-  }
-  if (ET->isFloat16Type() || ET == getContext().FloatTy || ET->isBFloat16Type())
-    return {SSE};
-  if (ET == getContext().DoubleTy)
-    return {SSE, SSE};
-  if (ET == getContext().LongDoubleTy) {
+      Classes = {Integer};
+    else if (Size <= 128)
+      Classes = {Integer, Integer};
+    else
+      llvm_unreachable("unexpected complex integer type size");
+  } else if (ET->isFloat16Type() || ET == getContext().FloatTy ||
+             ET->isBFloat16Type()) {
+    Classes = {SSE};
+  } else if (ET == getContext().DoubleTy) {
+    Classes = {SSE, SSE};
+  } else if (ET == getContext().LongDoubleTy) {
     const llvm::fltSemantics *LDF = &getTarget().getLongDoubleFormat();
     if (LDF == &llvm::APFloat::IEEEquad())
-      return {Memory};
-    if (LDF == &llvm::APFloat::x87DoubleExtended())
-      return {ComplexX87};
-    if (LDF == &llvm::APFloat::IEEEdouble())
-      return {SSE, SSE};
-    llvm_unreachable("unexpected long double representation!");
+      Classes = {Memory};
+    else if (LDF == &llvm::APFloat::x87DoubleExtended())
+      Classes = {ComplexX87};
+    else if (LDF == &llvm::APFloat::IEEEdouble())
+      Classes = {SSE, SSE};
+    else
+      llvm_unreachable("unexpected long double representation!");
+  } else if (ET->isFloat128Type()) {
+    Classes = {Memory};
+  } else {
+    llvm_unreachable("unexpected complex element type");
   }
-  if (ET->isFloat128Type())
-    return {Memory};
-  llvm_unreachable("unexpected complex element type");
+
+  uint64_t EBReal = Offset / 64;
+  uint64_t EBImag = (Offset + ElementSize) / 64;
+  if (Classes.Hi == NoClass && EBReal != EBImag)
+    Classes.Hi = Classes.Lo;
+  return Classes;
 }
 
 X86_64ABIInfo::ClassPair
@@ -2060,6 +2083,13 @@ X86_64ABIInfo::getBitIntTypeClassification(const BitIntType *BT) {
   return {Memory};
 }
 
+X86_64ABIInfo::ClassPair
+X86_64ABIInfo::getClassPairForSpan(uint64_t Offset, uint64_t Size, Class C) {
+  assert(Size && Offset % 64 + Size <= 128 &&
+         "span must touch no more than two eightbytes");
+  return {C, Offset % 64 + Size > 64 ? C : NoClass};
+}
+
 X86_64ABIInfo::VectorTypeInfo
 X86_64ABIInfo::getVectorTypeInfo(const VectorType *VT,
                                  const MatrixType *MT) const {
@@ -2077,6 +2107,16 @@ X86_64ABIInfo::getVectorTypeInfo(const VectorType *VT,
           IsInt128Element};
 }
 
+bool X86_64ABIInfo::isArrayPassedInMemory(const ConstantArrayType *AT,
+                                          uint64_t Offset, uint64_t Size,
+                                          bool IsRegCall) const {
+  // AMD64-ABI 3.2.3p2: Rule 1. Objects larger than eight eightbytes or
+  // containing unaligned fields have class MEMORY. Regcall does not impose the
+  // SysV size limit.
+  return (!IsRegCall && Size > 512) ||
+         Offset % getContext().getTypeAlign(AT->getElementType());
+}
+
 bool X86_64ABIInfo::isRecordPassedInMemory(const RecordType *RT,
                                            uint64_t Size) const {
   // AMD64-ABI 3.2.3p2: Rule 1. Objects larger than eight eightbytes have class
@@ -2174,12 +2214,6 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
     EightBytes[Offset / 64] = C;
   };
 
-  auto SetAdjacentEightBytes = [&](uint64_t Offset, Class LoClass,
-                                   Class HiClass) {
-    SetEightByte(Offset, LoClass);
-    SetEightByte(llvm::alignDown(Offset, uint64_t(64)) + 64, HiClass);
-  };
-
   auto SetClassPair = [&](uint64_t Offset, ClassPair Classes) {
     if (Classes.Lo == Memory || Classes.Hi == Memory) {
       ClassifyAsMemory();
@@ -2191,12 +2225,6 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
       SetEightByte(llvm::alignDown(Offset, uint64_t(64)) + 64, Classes.Hi);
   };
 
-  auto SetTouchedEightBytes = [&](uint64_t Size, Class C) {
-    for (uint64_t Offset = OffsetBase, End = OffsetBase + Size; Offset < End;
-         Offset = llvm::alignDown(Offset, uint64_t(64)) + 64)
-      SetEightByte(Offset, C);
-  };
-
   // End Helpers
 
   // Resize the EigthBytes vector if needed to fit the type
@@ -2241,7 +2269,6 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
 
   // Enums
   if (const auto *ED = Ty->getAsEnumDecl()) {
-    // Classify the underlying integer type.
     classifyClang24(ED->getIntegerType(), OffsetBase, EightBytes, isNamedArg);
     return;
   }
@@ -2255,7 +2282,7 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
   // C++ pointer-to-member types have integer-class storage: data member
   // pointers are one ptrdiff_t, and member function pointers are two.
   if (Ty->isMemberPointerType()) {
-    SetTouchedEightBytes(Size, Integer);
+    SetClassPair(OffsetBase, getClassPairForSpan(OffsetBase, Size, Integer));
     return;
   }
 
@@ -2331,17 +2358,7 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
 
   // Complex types
   if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
-    SetClassPair(OffsetBase, getComplexTypeClassification(CT));
-
-    // If this complex type crosses an eightbyte boundary then it
-    // should be split.
-    QualType ET = getContext().getCanonicalType(CT->getElementType());
-    uint64_t EB_Real = (OffsetBase) / 64;
-    uint64_t EB_Imag = (OffsetBase + getContext().getTypeSize(ET)) / 64;
-    if (EightBytes[EB_Imag] == NoClass && EB_Real != EB_Imag)
-      SetAdjacentEightBytes(OffsetBase, EightBytes[EB_Real],
-                            EightBytes[EB_Real]);
-
+    SetClassPair(OffsetBase, getComplexTypeClassification(CT, OffsetBase));
     return;
   }
 
@@ -2353,26 +2370,12 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
 
   // Arrays
   if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
-    // Arrays are treated like structures.
-
-    // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
-    // than eight eightbytes, ..., it has class MEMORY.
-    if (Size > 512) {
-      ClassifyAsMemory();
-      return;
-    }
-
-    // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
-    // fields, it has class MEMORY.
-    //
-    // Only need to check alignment of array base.
-    if (OffsetBase % getContext().getTypeAlign(AT->getElementType())) {
+    if (isArrayPassedInMemory(AT, OffsetBase, Size,
+                              /*IsRegCall=*/false)) {
       ClassifyAsMemory();
       return;
     }
 
-    // Otherwise implement simplified merge. We could be smarter about
-    // this, but it isn't worth it and would be harder to verify.
     uint64_t EltSize = getContext().getTypeSize(AT->getElementType());
     uint64_t ArraySize = AT->getZExtSize();
 
@@ -2400,23 +2403,19 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
     const RecordDecl *RD = RT->getDecl()->getDefinitionOrSelf();
     const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
 
-    // If this is a C++ record, classify the bases first.
+    // Classify bases followed by fields, merging each component's eightbyte
+    // classes. Bit-fields are handled directly because they may straddle
+    // eightbyte boundaries.
     if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
       for (const auto &I : CXXRD->bases()) {
         assert(!I.isVirtual() && !I.getType()->isDependentType() &&
                "Unexpected base class!");
         const auto *Base = I.getType()->castAsCXXRecordDecl();
 
-        // Classify this field.
-        //
-        // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds
-        // a single eightbyte, each is classified separately. Each
-        // eightbyte gets initialized to class NO_CLASS.
         SmallVector<Class> BaseEightBytes;
         uint64_t Offset =
             OffsetBase + getContext().toBits(Layout.getBaseClassOffset(Base));
         classifyClang24(I.getType(), Offset % 64, BaseEightBytes, isNamedArg);
-        // If the base is memory the whole record will be passed as memory
         if (BaseEightBytes[0] == Memory) {
           ClassifyAsMemory();
           return;
@@ -2425,7 +2424,6 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
       }
     }
 
-    // Classify the fields one at a time, merging the results.
     unsigned idx = 0;
 
     for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
@@ -2440,22 +2438,13 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
 
       bool IsInMemory =
           Offset % getContext().getTypeAlign(i->getType().getCanonicalType());
-      // Note, skip this test for bit-fields, see below.
+      // Bit-fields may be unaligned and are handled below.
       if (!BitField && IsInMemory) {
         ClassifyAsMemory();
         PostMerge();
         return;
       }
 
-      // Classify this field.
-      //
-      // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate
-      // exceeds a single eightbyte, each is classified
-      // separately. Each eightbyte gets initialized to class
-      // NO_CLASS.
-      // Bit-fields require special handling, they do not force the
-      // structure to be passed in memory even if unaligned, and
-      // therefore they can straddle an eightbyte.
       if (BitField) {
         assert(!i->isZeroLengthBitField());
         uint64_t BitSize = i->getBitWidthValue();
@@ -2549,25 +2538,8 @@ void X86_64ABIInfo::classifyClang23(QualType Ty, uint64_t OffsetBase,
   }
 
   if (Ty->isMemberPointerType()) {
-    if (Ty->isMemberFunctionPointerType()) {
-      if (Has64BitPointers) {
-        // If Has64BitPointers, this is an {i64, i64}, so classify both
-        // Lo and Hi now.
-        Lo = Hi = Integer;
-      } else {
-        // Otherwise, with 32-bit pointers, this is an {i32, i32}. If that
-        // straddles an eightbyte boundary, Hi should be classified as well.
-        uint64_t EB_FuncPtr = (OffsetBase) / 64;
-        uint64_t EB_ThisAdj = (OffsetBase + 64 - 1) / 64;
-        if (EB_FuncPtr != EB_ThisAdj) {
-          Lo = Hi = Integer;
-        } else {
-          Current = Integer;
-        }
-      }
-    } else {
-      Current = Integer;
-    }
+    SetClassPair(
+        getClassPairForSpan(OffsetBase, getContext().getTypeSize(Ty), Integer));
     return;
   }
 
@@ -2635,16 +2607,7 @@ void X86_64ABIInfo::classifyClang23(QualType Ty, uint64_t OffsetBase,
   }
 
   if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
-    SetClassPair(getComplexTypeClassification(CT));
-
-    // If this complex type crosses an eightbyte boundary then it
-    // should be split.
-    QualType ET = getContext().getCanonicalType(CT->getElementType());
-    uint64_t EB_Real = (OffsetBase) / 64;
-    uint64_t EB_Imag = (OffsetBase + getContext().getTypeSize(ET)) / 64;
-    if (Hi == NoClass && EB_Real != EB_Imag)
-      Hi = Lo;
-
+    SetClassPair(getComplexTypeClassification(CT, OffsetBase));
     return;
   }
 
@@ -2655,21 +2618,8 @@ void X86_64ABIInfo::classifyClang23(QualType Ty, uint64_t OffsetBase,
 
   if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
     // Arrays are treated like structures.
-
     uint64_t Size = getContext().getTypeSize(Ty);
-
-    // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
-    // than eight eightbytes, ..., it has class MEMORY.
-    // regcall ABI doesn't have limitation to an object. The only limitation
-    // is the free registers, which will be checked in computeInfo.
-    if (!IsRegCall && Size > 512)
-      return;
-
-    // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
-    // fields, it has class MEMORY.
-    //
-    // Only need to check alignment of array base.
-    if (OffsetBase % getContext().getTypeAlign(AT->getElementType()))
+    if (isArrayPassedInMemory(AT, OffsetBase, Size, IsRegCall))
       return;
 
     // Otherwise implement simplified merge. We could be smarter about

>From c6a52c13a11210ca46dff982ad9c9ef37bd1ea8e Mon Sep 17 00:00:00 2001
From: Benjamin Luke <benjamin.luke at sony.com>
Date: Sun, 13 Sep 2026 13:08:35 -0700
Subject: [PATCH 12/18] Add a test for missing enum abi coverage

---
 clang/test/CodeGen/X86/x86_64-enum-abi.c | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 clang/test/CodeGen/X86/x86_64-enum-abi.c

diff --git a/clang/test/CodeGen/X86/x86_64-enum-abi.c b/clang/test/CodeGen/X86/x86_64-enum-abi.c
new file mode 100644
index 00000000000000..f57b8611e8081c
--- /dev/null
+++ b/clang/test/CodeGen/X86/x86_64-enum-abi.c
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -fclang-abi-compat=23 -emit-llvm %s -o - | FileCheck %s
+
+enum E { A, B };
+
+struct EnumRecord {
+  enum E value;
+};
+
+// CHECK-LABEL: define dso_local void @take_enum_record(
+// CHECK-SAME: i32 %{{.*}})
+void take_enum_record(struct EnumRecord value) {}
+
+// CHECK-LABEL: define dso_local i32 @return_enum_record(
+struct EnumRecord return_enum_record(struct EnumRecord value) { return value; }

>From 73f7ee9f62828d33a5d4ab209d611e3867c187dd Mon Sep 17 00:00:00 2001
From: Benjamin Luke <benjamin.luke at sony.com>
Date: Sun, 13 Sep 2026 14:18:11 -0700
Subject: [PATCH 13/18] Fix the CI jobs

---
 clang/test/CodeGen/X86/x86_64-atomic-abi.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/test/CodeGen/X86/x86_64-atomic-abi.c b/clang/test/CodeGen/X86/x86_64-atomic-abi.c
index ada36a7cc7cdc5..599f505f7649a1 100644
--- a/clang/test/CodeGen/X86/x86_64-atomic-abi.c
+++ b/clang/test/CodeGen/X86/x86_64-atomic-abi.c
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefix=CLANG24
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fclang-abi-compat=23 %s -o - | FileCheck %s
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fexperimental-abi-lowering %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fexperimental-abi-lowering -fclang-abi-compat=23 %s -o - | FileCheck %s
 
 struct Floats {
   float a, b;

>From bd6421a758f9c3fe0f6f99956b7e241c310dbe3a Mon Sep 17 00:00:00 2001
From: Benjamin Luke <benjamin.luke at sony.com>
Date: Wed, 30 Sep 2026 12:01:01 -0700
Subject: [PATCH 14/18] Unify the clang23 and clang24 classifiers back into
 classify

---
 clang/lib/CodeGen/Targets/X86.cpp | 739 ++++++++++++++----------------
 1 file changed, 335 insertions(+), 404 deletions(-)

diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index 9c3e08132593e1..54d0401cde3ec6 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -1351,18 +1351,16 @@ class X86_64ABIInfo : public ABIInfo {
   /// responsible for placing them at the type's offset.
   ClassPair getBuiltinTypeClassification(const BuiltinType *BT) const;
 
-  /// Return the classes intrinsic to the complex type \param CT, including any
-  /// splitting required at \param Offset. The caller is responsible for class
-  /// placement.
-  ClassPair getComplexTypeClassification(const ComplexType *CT,
-                                         uint64_t Offset) const;
+  /// Return the classes that make up the complex type \param CT. The caller is
+  /// responsible for boundary splitting and class placement.
+  ClassPair getComplexTypeClassification(const ComplexType *CT) const;
 
   /// Return the classes intrinsic to the bit-precise integer type \param BT.
   /// The caller is responsible for class placement.
   static ClassPair getBitIntTypeClassification(const BitIntType *BT);
 
-  /// Return \param C for each eightbyte touched by the bit span beginning at
-  /// \param Offset. The result is relative to the first touched eightbyte.
+  /// Return \param C for each eightbyte touched by a bit span starting at
+  /// \param Offset. The span must touch at most two eightbytes.
   static ClassPair getClassPairForSpan(uint64_t Offset, uint64_t Size, Class C);
 
   /// Collect the canonical element type and shared classification properties
@@ -1383,10 +1381,9 @@ class X86_64ABIInfo : public ABIInfo {
   /// Determine the x86_64 register classes used to pass or return \p T.
   ///
   /// Each element of \p EightBytes describes an eightbyte in the containing
-  /// object. The Clang 24 classifier can produce an entry for every eightbyte,
-  /// while the legacy classifier represents the result using its first two
-  /// entries. NoClass denotes an unused eightbyte and Memory denotes a value
-  /// that must be passed in memory.
+  /// object. The current classifier can produce an entry for every eightbyte;
+  /// the legacy classifier uses the first two entries. NoClass denotes an
+  /// unused eightbyte and Memory denotes a value passed in memory.
   ///
   /// \param T The type to classify.
   /// \param OffsetBase The bit offset of \p T in its containing object.
@@ -1395,19 +1392,24 @@ class X86_64ABIInfo : public ABIInfo {
   /// \param EightBytes The resulting eightbyte classes.
   /// \param isNamedArg Whether the argument is named rather than part of a
   /// variadic argument list, as used in AMD64-ABI 3.5.7.
-  /// \param isRegCall Whether the calling convention is regcall. Regcall uses
-  /// the legacy classifier.
+  /// \param isRegCall Whether the calling convention is regcall.
+  /// \param useLegacy Whether to use the pre-Clang-24 classification rules.
+  /// This must be true for regcall, which retains the legacy ABI.
   void classify(QualType T, uint64_t OffsetBase,
                 SmallVectorImpl<Class> &EightBytes, bool isNamedArg,
-                bool isRegCall = false) const;
-
-  void classifyClang23(QualType T, uint64_t OffsetBase,
-                       SmallVectorImpl<Class> &EightBytes, bool isNamedArg,
-                       bool isRegCall) const;
-
-  void classifyClang24(QualType T, uint64_t OffsetBase,
-                       SmallVectorImpl<Class> &EightBytes,
-                       bool isNamedArg) const;
+                bool isRegCall, bool useLegacy) const;
+
+  void postMergeLegacy(SmallVectorImpl<Class> &EightBytes,
+                       unsigned AggregateSize) const;
+  void classifyLegacyVector(const VectorType *VT, uint64_t OffsetBase,
+                            SmallVectorImpl<Class> &EightBytes,
+                            bool isNamedArg) const;
+  void classifyLegacyArray(const ConstantArrayType *AT, uint64_t OffsetBase,
+                           uint64_t Size, SmallVectorImpl<Class> &EightBytes,
+                           bool isNamedArg, bool isRegCall) const;
+  void classifyLegacyRecord(const RecordType *RT, uint64_t OffsetBase,
+                            uint64_t Size, SmallVectorImpl<Class> &EightBytes,
+                            bool isNamedArg, bool isRegCall) const;
 
   llvm::Type *GetByteVectorType(QualType Ty) const;
   llvm::Type *GetSSETypeAtOffset(llvm::Type *IRType,
@@ -2032,10 +2034,8 @@ X86_64ABIInfo::getBuiltinTypeClassification(const BuiltinType *BT) const {
 }
 
 X86_64ABIInfo::ClassPair
-X86_64ABIInfo::getComplexTypeClassification(const ComplexType *CT,
-                                            uint64_t Offset) const {
+X86_64ABIInfo::getComplexTypeClassification(const ComplexType *CT) const {
   QualType ET = getContext().getCanonicalType(CT->getElementType());
-  uint64_t ElementSize = getContext().getTypeSize(ET);
   uint64_t Size = getContext().getTypeSize(QualType(CT, 0));
   ClassPair Classes;
 
@@ -2067,10 +2067,6 @@ X86_64ABIInfo::getComplexTypeClassification(const ComplexType *CT,
     llvm_unreachable("unexpected complex element type");
   }
 
-  uint64_t EBReal = Offset / 64;
-  uint64_t EBImag = (Offset + ElementSize) / 64;
-  if (Classes.Hi == NoClass && EBReal != EBImag)
-    Classes.Hi = Classes.Lo;
   return Classes;
 }
 
@@ -2133,22 +2129,247 @@ bool X86_64ABIInfo::isRecordPassedInMemory(const RecordType *RT,
   return RT->getDecl()->getDefinitionOrSelf()->hasFlexibleArrayMember();
 }
 
+void X86_64ABIInfo::postMergeLegacy(SmallVectorImpl<Class> &EightBytes,
+                                    unsigned AggregateSize) const {
+  assert(EightBytes.size() >= 2);
+  Class &Lo = EightBytes[0];
+  Class &Hi = EightBytes[1];
+
+  // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
+  // (a) MEMORY in either slot makes the whole argument MEMORY.
+  // (b) X87UP without a preceding X87 makes the argument MEMORY on platforms
+  //     that honor revision 0.98.
+  // (c) Aggregates wider than two eightbytes must be SSE followed by SSEUP.
+  // (d) SSEUP without a preceding SSE becomes SSE.
+  if (Hi == Memory)
+    Lo = Memory;
+  if (Hi == X87Up && Lo != X87 && honorsRevision0_98())
+    Lo = Memory;
+  if (AggregateSize > 128 && (Lo != SSE || Hi != SSEUp))
+    Lo = Memory;
+  if (Hi == SSEUp && Lo != SSE)
+    Hi = SSE;
+}
+
+void X86_64ABIInfo::classifyLegacyVector(const VectorType *VT,
+                                         uint64_t OffsetBase,
+                                         SmallVectorImpl<Class> &EightBytes,
+                                         bool isNamedArg) const {
+  assert(EightBytes.size() >= 2);
+  Class &Lo = EightBytes[0];
+  Class &Hi = EightBytes[1];
+  Class &Current = OffsetBase < 64 ? Lo : Hi;
+  uint64_t Size = getContext().getTypeSize(VT);
+  VectorTypeInfo Info = getVectorTypeInfo(VT);
+
+  if (Size == 1 || Size == 8 || Size == 16 || Size == 32) {
+    // Clang 23 classifies small vectors as INTEGER, including <1 x float>.
+    Current = Integer;
+    uint64_t EB_Lo = OffsetBase / 64;
+    uint64_t EB_Hi = (OffsetBase + Size - 1) / 64;
+    if (EB_Lo != EB_Hi)
+      Hi = Lo;
+  } else if (Size == 64) {
+    // A single double element retains the memory default.
+    if (Info.ElementType->isSpecificBuiltinType(BuiltinType::Double))
+      return;
+
+    if (!classifyIntegerMMXAsSSE() &&
+        (Info.ElementType->isSpecificBuiltinType(BuiltinType::LongLong) ||
+         Info.ElementType->isSpecificBuiltinType(BuiltinType::ULongLong) ||
+         Info.ElementType->isSpecificBuiltinType(BuiltinType::Long) ||
+         Info.ElementType->isSpecificBuiltinType(BuiltinType::ULong)))
+      Current = Integer;
+    else
+      Current = SSE;
+
+    if (OffsetBase && OffsetBase != 64)
+      Hi = Lo;
+  } else if (Size == 128 ||
+             (isNamedArg && Size <= getNativeVectorSizeForAVXABI(AVXLevel))) {
+    if (passInt128VectorsInMem() && Size != 128 && Info.IsInt128Element)
+      return;
+
+    // Wider vectors still use only the two legacy slots.
+    Lo = SSE;
+    Hi = SSEUp;
+  }
+}
+
+void X86_64ABIInfo::classifyLegacyArray(const ConstantArrayType *AT,
+                                        uint64_t OffsetBase, uint64_t Size,
+                                        SmallVectorImpl<Class> &EightBytes,
+                                        bool isNamedArg, bool isRegCall) const {
+  assert(EightBytes.size() >= 2);
+  Class &Lo = EightBytes[0];
+  Class &Hi = EightBytes[1];
+  Class &Current = OffsetBase < 64 ? Lo : Hi;
+
+  if (isArrayPassedInMemory(AT, OffsetBase, Size, isRegCall))
+    return;
+
+  Current = NoClass;
+  uint64_t EltSize = getContext().getTypeSize(AT->getElementType());
+  uint64_t ArraySize = AT->getZExtSize();
+
+  // Preserve the Clang 23 early return that leaves Current as NO_CLASS.
+  if (Size > 128 &&
+      (Size != EltSize || Size > getNativeVectorSizeForAVXABI(AVXLevel)))
+    return;
+
+  for (uint64_t I = 0, Offset = OffsetBase; I < ArraySize;
+       ++I, Offset += EltSize) {
+    SmallVector<Class> FieldEightBytes = {NoClass, NoClass};
+    classify(AT->getElementType(), Offset, FieldEightBytes, isNamedArg,
+             isRegCall, /*useLegacy=*/true);
+    Lo = merge(Lo, FieldEightBytes[0]);
+    Hi = merge(Hi, FieldEightBytes[1]);
+    if (Lo == Memory || Hi == Memory)
+      break;
+  }
+
+  postMergeLegacy(EightBytes, Size);
+  assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp array classification.");
+}
+
+void X86_64ABIInfo::classifyLegacyRecord(const RecordType *RT,
+                                         uint64_t OffsetBase, uint64_t Size,
+                                         SmallVectorImpl<Class> &EightBytes,
+                                         bool isNamedArg,
+                                         bool isRegCall) const {
+  assert(EightBytes.size() >= 2);
+  Class &Lo = EightBytes[0];
+  Class &Hi = EightBytes[1];
+  Class &Current = OffsetBase < 64 ? Lo : Hi;
+
+  if (isRecordPassedInMemory(RT, Size))
+    return;
+
+  const RecordDecl *RD = RT->getDecl()->getDefinitionOrSelf();
+  const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
+  Current = NoClass;
+
+  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
+    for (const auto &I : CXXRD->bases()) {
+      assert(!I.isVirtual() && !I.getType()->isDependentType() &&
+             "Unexpected base class!");
+      const auto *Base = I.getType()->castAsCXXRecordDecl();
+
+      SmallVector<Class> FieldEightBytes = {NoClass, NoClass};
+      uint64_t Offset =
+          OffsetBase + getContext().toBits(Layout.getBaseClassOffset(Base));
+      classify(I.getType(), Offset, FieldEightBytes, isNamedArg, isRegCall,
+               /*useLegacy=*/true);
+      // Preserve the legacy two-eightbyte merge for wide bases.
+      Lo = merge(Lo, FieldEightBytes[0]);
+      Hi = merge(Hi, FieldEightBytes[1]);
+      if (returnCXXRecordGreaterThan128InMem() &&
+          !isEmptyRecord(getContext(), I.getType(), true) &&
+          (Size > 128 && (Size != getContext().getTypeSize(I.getType()) ||
+                          Size > getNativeVectorSizeForAVXABI(AVXLevel)))) {
+        Lo = Memory;
+      }
+      if (Lo == Memory || Hi == Memory) {
+        postMergeLegacy(EightBytes, Size);
+        return;
+      }
+    }
+  }
+
+  unsigned Idx = 0;
+  bool UseClang11Compat = getContext().getLangOpts().isCompatibleWith(
+                              LangOptions::ClangABI::Ver11) ||
+                          getContext().getTargetInfo().getTriple().isPS();
+  bool ClassifyUnnamedBitFields =
+      getContext().getLangOpts().getClangABICompat() >
+          LangOptions::ClangABI::Ver23 &&
+      !getContext().getTargetInfo().getTriple().isPS();
+  bool IsUnion = RT->isUnionType() && !UseClang11Compat;
+
+  for (RecordDecl::field_iterator I = RD->field_begin(), E = RD->field_end();
+       I != E; ++I, ++Idx) {
+    uint64_t Offset = OffsetBase + Layout.getFieldOffset(Idx);
+    bool BitField = I->isBitField();
+
+    // Clang 23 treats all unnamed bit-fields as padding, including cases
+    // where skipping one leaves part of a wider access unit unclassified.
+    if (BitField && (ClassifyUnnamedBitFields ? I->isZeroLengthBitField()
+                                              : I->isUnnamedBitField()))
+      continue;
+
+    if (Size > 128 &&
+        ((!IsUnion && Size != getContext().getTypeSize(I->getType())) ||
+         Size > getNativeVectorSizeForAVXABI(AVXLevel))) {
+      Lo = Memory;
+      postMergeLegacy(EightBytes, Size);
+      return;
+    }
+
+    bool IsInMemory =
+        Offset % getContext().getTypeAlign(I->getType().getCanonicalType());
+    if (!BitField && IsInMemory) {
+      Lo = Memory;
+      postMergeLegacy(EightBytes, Size);
+      return;
+    }
+
+    SmallVector<Class> FieldEightBytes = {NoClass, NoClass};
+    if (BitField) {
+      assert(ClassifyUnnamedBitFields ? !I->isZeroLengthBitField()
+                                      : !I->isUnnamedBitField());
+      uint64_t BitSize = I->getBitWidthValue();
+      uint64_t EB_Lo = Offset / 64;
+      uint64_t EB_Hi = (Offset + BitSize - 1) / 64;
+
+      if (EB_Lo) {
+        assert(EB_Hi == EB_Lo && "Invalid classification, type > 16 bytes.");
+        FieldEightBytes[0] = NoClass;
+        FieldEightBytes[1] = Integer;
+      } else {
+        FieldEightBytes[0] = Integer;
+        FieldEightBytes[1] = EB_Hi ? Integer : NoClass;
+      }
+    } else {
+      classify(I->getType(), Offset, FieldEightBytes, isNamedArg, isRegCall,
+               /*useLegacy=*/true);
+    }
+    Lo = merge(Lo, FieldEightBytes[0]);
+    Hi = merge(Hi, FieldEightBytes[1]);
+    if (Lo == Memory || Hi == Memory)
+      break;
+  }
+
+  postMergeLegacy(EightBytes, Size);
+}
+
 void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
                              SmallVectorImpl<Class> &EightBytes,
-                             bool isNamedArg, bool IsRegCall) const {
-  if (useLegacyClassificationAlgorithm() || IsRegCall) {
-    // RegCall is a separate ABI from the SysV psABI. Keep its classification
-    // on the pre-Clang-24 rules so this SysV classifier refactor
-    // does not silently break ABI.
-    classifyClang23(Ty, OffsetBase, EightBytes, isNamedArg, IsRegCall);
+                             bool isNamedArg, bool isRegCall,
+                             bool useLegacy) const {
+
+  uint64_t Size = getContext().getTypeSize(Ty);
+  if (useLegacy) {
+    // Top-level legacy calls need two slots; recursive calls provide them.
+    if (EightBytes.empty())
+      EightBytes.resize(2, NoClass);
   } else {
-    classifyClang24(Ty, OffsetBase, EightBytes, isNamedArg);
+    unsigned Required = (OffsetBase + Size + 63) / 64;
+    if (EightBytes.size() < Required)
+      EightBytes.resize(Required, NoClass);
+
+    if (Size == 0) {
+      // No need to classify empty types, return NoClass
+      EightBytes.resize(1, NoClass);
+      return;
+    }
   }
-}
 
-void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
-                                    SmallVectorImpl<Class> &EightBytes,
-                                    bool isNamedArg) const {
+  auto Lo = [&]() -> Class & { return EightBytes[0]; };
+  auto Hi = [&]() -> Class & { return EightBytes[1]; };
+  auto Current = [&]() -> Class & { return OffsetBase < 64 ? Lo() : Hi(); };
+  if (useLegacy)
+    Current() = Memory;
+
   // Helpers
   auto ClassifyAsMemory = [&]() {
     if (EightBytes.empty())
@@ -2215,49 +2436,48 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
   };
 
   auto SetClassPair = [&](uint64_t Offset, ClassPair Classes) {
-    if (Classes.Lo == Memory || Classes.Hi == Memory) {
-      ClassifyAsMemory();
-      return;
+    if (useLegacy) {
+      if (Classes.Hi == NoClass) {
+        Current() = Classes.Lo;
+        return;
+      }
+      Lo() = Classes.Lo;
+      Hi() = Classes.Hi;
+    } else {
+      if (Classes.Lo == Memory || Classes.Hi == Memory) {
+        ClassifyAsMemory();
+        return;
+      }
+      if (Classes.Lo != NoClass)
+        SetEightByte(Offset, Classes.Lo);
+      if (Classes.Hi != NoClass)
+        SetEightByte(llvm::alignDown(Offset, uint64_t(64)) + 64, Classes.Hi);
     }
-    if (Classes.Lo != NoClass)
-      SetEightByte(Offset, Classes.Lo);
-    if (Classes.Hi != NoClass)
-      SetEightByte(llvm::alignDown(Offset, uint64_t(64)) + 64, Classes.Hi);
   };
 
   // End Helpers
 
-  // Resize the EigthBytes vector if needed to fit the type
-  uint64_t Size = getContext().getTypeSize(Ty);
-  unsigned Required = (OffsetBase + Size + 63) / 64;
-  if (EightBytes.size() < Required)
-    EightBytes.resize(Required, NoClass);
-
-  if (Size == 0) {
-    EightBytes.resize(1, NoClass);
-    return; // No need to classify empty types, return NoClass
-  }
-
   // Type classification begin
 
   // Overflow behaviour types
-  if (const OverflowBehaviorType *OBT = Ty->getAs<OverflowBehaviorType>()) {
-    classifyClang24(OBT->getUnderlyingType(), OffsetBase, EightBytes,
-                    isNamedArg);
+  if (const OverflowBehaviorType *OBT = Ty->getAs<OverflowBehaviorType>();
+      !useLegacy && OBT) {
+    classify(OBT->getUnderlyingType(), OffsetBase, EightBytes, isNamedArg,
+             isRegCall, useLegacy);
     return;
   }
 
   // Atomic types usually classify the same way as their value type. If Clang
   // promotes the atomic storage size, fall back to memory so the full storage
   // representation is preserved across the ABI boundary.
-  if (const AtomicType *AT = Ty->getAs<AtomicType>()) {
+  if (const AtomicType *AT = Ty->getAs<AtomicType>(); !useLegacy && AT) {
     QualType ValueTy = AT->getValueType();
     if (Size != getContext().getTypeSize(ValueTy)) {
       ClassifyAsMemory();
       return;
     }
 
-    classifyClang24(ValueTy, OffsetBase, EightBytes, isNamedArg);
+    classify(ValueTy, OffsetBase, EightBytes, isNamedArg, isRegCall, useLegacy);
     return;
   }
 
@@ -2269,13 +2489,18 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
 
   // Enums
   if (const auto *ED = Ty->getAsEnumDecl()) {
-    classifyClang24(ED->getIntegerType(), OffsetBase, EightBytes, isNamedArg);
+    classify(ED->getIntegerType(), OffsetBase, EightBytes, isNamedArg,
+             isRegCall, useLegacy);
     return;
   }
 
   // Pointers and OpenCL pipes have pointer-sized runtime representations.
-  if (Ty->hasPointerRepresentation() || Ty->isPipeType()) {
-    SetEightByte(OffsetBase, Integer);
+  if (Ty->hasPointerRepresentation() || (Ty->isPipeType() && !useLegacy)) {
+    if (useLegacy) {
+      Current() = Integer;
+    } else {
+      SetEightByte(OffsetBase, Integer);
+    }
     return;
   }
 
@@ -2291,7 +2516,12 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
   // same element type.
   const VectorType *VT = Ty->getAs<VectorType>();
   const MatrixType *MT = Ty->getAs<MatrixType>();
-  if (VT || MT) {
+  if (VT || (MT && !useLegacy)) {
+    if (useLegacy) {
+      classifyLegacyVector(VT, OffsetBase, EightBytes, isNamedArg);
+      return;
+    }
+
     VectorTypeInfo Info = getVectorTypeInfo(VT, MT);
 
     auto SetEightByteClasses = [&](Class FirstClass, Class RestClass) {
@@ -2358,7 +2588,23 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
 
   // Complex types
   if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
-    SetClassPair(OffsetBase, getComplexTypeClassification(CT, OffsetBase));
+    SetClassPair(OffsetBase, getComplexTypeClassification(CT));
+
+    // The complex type's initial class pair may leave Hi as NoClass even when
+    // its components start in different eightbytes. Fill that slot when needed.
+    QualType ET = getContext().getCanonicalType(CT->getElementType());
+    uint64_t EBReal = OffsetBase / 64;
+    uint64_t EBImag = (OffsetBase + getContext().getTypeSize(ET)) / 64;
+    if (EBReal == EBImag)
+      return;
+    if (useLegacy) {
+      if (Hi() == NoClass)
+        Hi() = Lo();
+    } else {
+      if (EightBytes[EBImag] == NoClass)
+        SetEightByte(llvm::alignDown(OffsetBase, uint64_t(64)) + 64,
+                     EightBytes[EBReal]);
+    }
     return;
   }
 
@@ -2370,6 +2616,11 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
 
   // Arrays
   if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
+    if (useLegacy) {
+      classifyLegacyArray(AT, OffsetBase, Size, EightBytes, isNamedArg,
+                          isRegCall);
+      return;
+    }
     if (isArrayPassedInMemory(AT, OffsetBase, Size,
                               /*IsRegCall=*/false)) {
       ClassifyAsMemory();
@@ -2382,8 +2633,8 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
     for (uint64_t i = 0, Offset = OffsetBase; i < ArraySize;
          ++i, Offset += EltSize) {
       SmallVector<Class> ArrayElEightBytes;
-      classifyClang24(AT->getElementType(), Offset % 64, ArrayElEightBytes,
-                      isNamedArg);
+      classify(AT->getElementType(), Offset % 64, ArrayElEightBytes, isNamedArg,
+               isRegCall, useLegacy);
       MergeIntoAdjacentEightBytes(Offset, ArrayElEightBytes);
       if (llvm::is_contained(EightBytes, Memory))
         break;
@@ -2395,6 +2646,12 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
 
   // CXX Record Type
   if (const RecordType *RT = Ty->getAsCanonical<RecordType>()) {
+    if (useLegacy) {
+      classifyLegacyRecord(RT, OffsetBase, Size, EightBytes, isNamedArg,
+                           isRegCall);
+      return;
+    }
+
     if (isRecordPassedInMemory(RT, Size)) {
       ClassifyAsMemory();
       return;
@@ -2415,7 +2672,8 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
         SmallVector<Class> BaseEightBytes;
         uint64_t Offset =
             OffsetBase + getContext().toBits(Layout.getBaseClassOffset(Base));
-        classifyClang24(I.getType(), Offset % 64, BaseEightBytes, isNamedArg);
+        classify(I.getType(), Offset % 64, BaseEightBytes, isNamedArg,
+                 isRegCall, useLegacy);
         if (BaseEightBytes[0] == Memory) {
           ClassifyAsMemory();
           return;
@@ -2454,7 +2712,8 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
           MergeIntoEightByte(BitOffset, Integer);
       } else {
         SmallVector<Class> FieldEightBytes;
-        classifyClang24(i->getType(), Offset % 64, FieldEightBytes, isNamedArg);
+        classify(i->getType(), Offset % 64, FieldEightBytes, isNamedArg,
+                 isRegCall, useLegacy);
         MergeIntoAdjacentEightBytes(Offset, FieldEightBytes);
       }
 
@@ -2463,338 +2722,8 @@ void X86_64ABIInfo::classifyClang24(QualType Ty, uint64_t OffsetBase,
     }
 
     PostMerge();
-  }
-}
-
-void X86_64ABIInfo::classifyClang23(QualType Ty, uint64_t OffsetBase,
-                                    SmallVectorImpl<Class> &EightBytes,
-                                    bool isNamedArg, bool IsRegCall) const {
-  // FIXME: Some of the split computations are wrong; unaligned vectors
-  // shouldn't be passed in registers for example, so there is no chance they
-  // can straddle an eightbyte. Verify & simplify.
-  if (EightBytes.empty())
-    EightBytes.resize(2, NoClass);
-
-  Class &Lo = EightBytes[0];
-  Class &Hi = EightBytes[1];
-  auto PostMerge = [&](unsigned AggregateSize) {
-    // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
-    //
-    // (a) If one of the classes is Memory, the whole argument is passed in
-    //     memory.
-    //
-    // (b) If X87UP is not preceded by X87, the whole argument is passed in
-    //     memory.
-    //
-    // (c) If the size of the aggregate exceeds two eightbytes and the first
-    //     eightbyte isn't SSE or any other eightbyte isn't SSEUP, the whole
-    //     argument is passed in memory. NOTE: This is necessary to keep the
-    //     ABI working for processors that don't support the __m256 type.
-    //
-    // (d) If SSEUP is not preceded by SSE or SSEUP, it is converted to SSE.
-    //
-    // Some of these are enforced by the merging logic. Others can arise only
-    // with unions; for example:
-    //   union { _Complex double; unsigned; }
-    //
-    // Note that clauses (b) and (c) were added in 0.98.
-    if (Hi == Memory)
-      Lo = Memory;
-    if (Hi == X87Up && Lo != X87 && honorsRevision0_98())
-      Lo = Memory;
-    if (AggregateSize > 128 && (Lo != SSE || Hi != SSEUp))
-      Lo = Memory;
-    if (Hi == SSEUp && Lo != SSE)
-      Hi = SSE;
-  };
-
-  Class &Current = OffsetBase < 64 ? Lo : Hi;
-  Current = Memory;
-
-  auto SetClassPair = [&](ClassPair Classes) {
-    if (Classes.Hi == NoClass) {
-      Current = Classes.Lo;
-      return;
-    }
-    Lo = Classes.Lo;
-    Hi = Classes.Hi;
-  };
-
-  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
-    SetClassPair(getBuiltinTypeClassification(BT));
-    return;
-  }
-
-  if (const auto *ED = Ty->getAsEnumDecl()) {
-    // Classify the underlying integer type.
-    classifyClang23(ED->getIntegerType(), OffsetBase, EightBytes, isNamedArg,
-                    IsRegCall);
-    return;
-  }
-
-  if (Ty->hasPointerRepresentation()) {
-    Current = Integer;
-    return;
-  }
-
-  if (Ty->isMemberPointerType()) {
-    SetClassPair(
-        getClassPairForSpan(OffsetBase, getContext().getTypeSize(Ty), Integer));
-    return;
-  }
-
-  if (const VectorType *VT = Ty->getAs<VectorType>()) {
-    uint64_t Size = getContext().getTypeSize(VT);
-    VectorTypeInfo Info = getVectorTypeInfo(VT);
-    if (Size == 1 || Size == 8 || Size == 16 || Size == 32) {
-      // gcc passes the following as integer:
-      // 4 bytes - <4 x char>, <2 x short>, <1 x int>, <1 x float>
-      // 2 bytes - <2 x char>, <1 x short>
-      // 1 byte  - <1 x char>
-      Current = Integer;
-
-      // If this type crosses an eightbyte boundary, it should be
-      // split.
-      uint64_t EB_Lo = (OffsetBase) / 64;
-      uint64_t EB_Hi = (OffsetBase + Size - 1) / 64;
-      if (EB_Lo != EB_Hi)
-        Hi = Lo;
-    } else if (Size == 64) {
-      // gcc passes <1 x double> in memory. :(
-      if (Info.ElementType->isSpecificBuiltinType(BuiltinType::Double))
-        return;
-
-      // gcc passes <1 x long long> as SSE but clang used to unconditionally
-      // pass them as integer.  For platforms where clang is the de facto
-      // platform compiler, we must continue to use integer.
-      if (!classifyIntegerMMXAsSSE() &&
-          (Info.ElementType->isSpecificBuiltinType(BuiltinType::LongLong) ||
-           Info.ElementType->isSpecificBuiltinType(BuiltinType::ULongLong) ||
-           Info.ElementType->isSpecificBuiltinType(BuiltinType::Long) ||
-           Info.ElementType->isSpecificBuiltinType(BuiltinType::ULong)))
-        Current = Integer;
-      else
-        Current = SSE;
-
-      // If this type crosses an eightbyte boundary, it should be
-      // split.
-      if (OffsetBase && OffsetBase != 64)
-        Hi = Lo;
-    } else if (Size == 128 ||
-               (isNamedArg && Size <= getNativeVectorSizeForAVXABI(AVXLevel))) {
-      // gcc passes 256 and 512 bit <X x __int128> vectors in memory. :(
-      if (passInt128VectorsInMem() && Size != 128 && Info.IsInt128Element)
-        return;
-
-      // Arguments of 256-bits are split into four eightbyte chunks. The
-      // least significant one belongs to class SSE and all the others to
-      // class SSEUP. The original Lo and Hi design considers that types can't
-      // be greater than 128-bits, so a 64-bit split in Hi and Lo makes sense.
-      // This design isn't correct for 256-bits, but since there're no cases
-      // where the upper parts would need to be inspected, avoid adding
-      // complexity and just consider Hi to match the 64-256 part.
-      //
-      // Note that per 3.5.7 of AMD64-ABI, 256-bit args are only passed in
-      // registers if they are "named", i.e. not part of the "..." of a
-      // variadic function.
-      //
-      // Similarly, per 3.2.3. of the AVX512 draft, 512-bits ("named") args
-      // are split into eight eightbyte chunks, one SSE and seven SSEUP.
-      Lo = SSE;
-      Hi = SSEUp;
-    }
-    return;
-  }
-
-  if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
-    SetClassPair(getComplexTypeClassification(CT, OffsetBase));
-    return;
-  }
-
-  if (const auto *EITy = Ty->getAs<BitIntType>()) {
-    SetClassPair(getBitIntTypeClassification(EITy));
-    return;
-  }
-
-  if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
-    // Arrays are treated like structures.
-    uint64_t Size = getContext().getTypeSize(Ty);
-    if (isArrayPassedInMemory(AT, OffsetBase, Size, IsRegCall))
-      return;
-
-    // Otherwise implement simplified merge. We could be smarter about
-    // this, but it isn't worth it and would be harder to verify.
-    Current = NoClass;
-    uint64_t EltSize = getContext().getTypeSize(AT->getElementType());
-    uint64_t ArraySize = AT->getZExtSize();
-
-    // The only case a 256-bit wide vector could be used is when the array
-    // contains a single 256-bit element. Since Lo and Hi logic isn't extended
-    // to work for sizes wider than 128, early check and fallback to memory.
-    //
-    // Note: There is a bug here, the assumption is that this is returning
-    // "Memory" but actually it is returning "NoClass" because Current is set to
-    // NoClass above. Fixed in the Clang 24 implementation.
-    if (Size > 128 &&
-        (Size != EltSize || Size > getNativeVectorSizeForAVXABI(AVXLevel)))
-      return;
-
-    for (uint64_t i = 0, Offset = OffsetBase; i < ArraySize;
-         ++i, Offset += EltSize) {
-      SmallVector<Class> FieldEightBytes = {NoClass, NoClass};
-      classifyClang23(AT->getElementType(), Offset, FieldEightBytes, isNamedArg,
-                      IsRegCall);
-      Lo = merge(Lo, FieldEightBytes[0]);
-      Hi = merge(Hi, FieldEightBytes[1]);
-      if (Lo == Memory || Hi == Memory)
-        break;
-    }
-
-    PostMerge(Size);
-    assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp array classification.");
     return;
   }
-
-  if (const RecordType *RT = Ty->getAsCanonical<RecordType>()) {
-    uint64_t Size = getContext().getTypeSize(Ty);
-    if (isRecordPassedInMemory(RT, Size))
-      return;
-
-    const RecordDecl *RD = RT->getDecl()->getDefinitionOrSelf();
-    const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
-
-    // Reset Lo class, this will be recomputed.
-    Current = NoClass;
-
-    // If this is a C++ record, classify the bases first.
-    if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
-      for (const auto &I : CXXRD->bases()) {
-        assert(!I.isVirtual() && !I.getType()->isDependentType() &&
-               "Unexpected base class!");
-        const auto *Base = I.getType()->castAsCXXRecordDecl();
-
-        // Classify this field.
-        //
-        // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate exceeds a
-        // single eightbyte, each is classified separately. Each eightbyte
-        // gets initialized to class NO_CLASS.
-        SmallVector<Class> FieldEightBytes = {NoClass, NoClass};
-        uint64_t Offset =
-            OffsetBase + getContext().toBits(Layout.getBaseClassOffset(Base));
-        classifyClang23(I.getType(), Offset, FieldEightBytes, isNamedArg,
-                        IsRegCall);
-        // Note: There is a bug here for types larger than 128bits. Doesn't
-        // properly implement the SYSV classification algorithm. Fixed in
-        // Clang 24 implementation.
-        Lo = merge(Lo, FieldEightBytes[0]);
-        Hi = merge(Hi, FieldEightBytes[1]);
-        if (returnCXXRecordGreaterThan128InMem() &&
-            !isEmptyRecord(getContext(), I.getType(), true) &&
-            (Size > 128 && (Size != getContext().getTypeSize(I.getType()) ||
-                            Size > getNativeVectorSizeForAVXABI(AVXLevel)))) {
-          // The only case a 256(or 512)-bit wide vector could be used to
-          // return is when CXX record contains a single 256(or 512)-bit
-          // element.
-          Lo = Memory;
-        }
-        if (Lo == Memory || Hi == Memory) {
-          PostMerge(Size);
-          return;
-        }
-      }
-    }
-
-    // Classify the fields one at a time, merging the results.
-    unsigned idx = 0;
-    bool UseClang11Compat = getContext().getLangOpts().isCompatibleWith(
-                                LangOptions::ClangABI::Ver11) ||
-                            getContext().getTargetInfo().getTriple().isPS();
-    bool ClassifyUnnamedBitFields =
-        getContext().getLangOpts().getClangABICompat() >
-            LangOptions::ClangABI::Ver23 &&
-        !getContext().getTargetInfo().getTriple().isPS();
-    bool IsUnion = RT->isUnionType() && !UseClang11Compat;
-
-    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
-         i != e; ++i, ++idx) {
-      uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
-      bool BitField = i->isBitField();
-
-      // Ignore padding bit-fields. Normally only zero-length bit-fields are
-      // padding, but under -fclang-abi-compat=23 every unnamed bit-field is,
-      // faithfully reproducing Clang 23 -- including its crash on aggregates
-      // where skipping one leaves part of a wider access unit (e.g. an
-      // __int128 bit-field run) unclassified.
-      if (BitField && (ClassifyUnnamedBitFields ? i->isZeroLengthBitField()
-                                                : i->isUnnamedBitField()))
-        continue;
-
-      // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger than
-      // eight eightbytes, or it contains unaligned fields, it has class
-      // MEMORY.
-      //
-      // The only case a 256-bit or a 512-bit wide vector could be used is
-      // when the struct contains a single 256-bit or 512-bit element. Early
-      // check and fallback to memory.
-      //
-      // FIXME: Extended the Lo and Hi logic properly to work for size wider
-      // than 128.
-      if (Size > 128 &&
-          ((!IsUnion && Size != getContext().getTypeSize(i->getType())) ||
-           Size > getNativeVectorSizeForAVXABI(AVXLevel))) {
-        Lo = Memory;
-        PostMerge(Size);
-        return;
-      }
-
-      bool IsInMemory =
-          Offset % getContext().getTypeAlign(i->getType().getCanonicalType());
-      // Note, skip this test for bit-fields, see below.
-      if (!BitField && IsInMemory) {
-        Lo = Memory;
-        PostMerge(Size);
-        return;
-      }
-
-      // Classify this field.
-      //
-      // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate
-      // exceeds a single eightbyte, each is classified
-      // separately. Each eightbyte gets initialized to class
-      // NO_CLASS.
-      SmallVector<Class> FieldEightBytes = {NoClass, NoClass};
-
-      // Bit-fields require special handling, they do not force the
-      // structure to be passed in memory even if unaligned, and
-      // therefore they can straddle an eightbyte.
-      if (BitField) {
-        assert(ClassifyUnnamedBitFields ? !i->isZeroLengthBitField()
-                                        : !i->isUnnamedBitField());
-        uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
-        uint64_t Size = i->getBitWidthValue();
-
-        uint64_t EB_Lo = Offset / 64;
-        uint64_t EB_Hi = (Offset + Size - 1) / 64;
-
-        if (EB_Lo) {
-          assert(EB_Hi == EB_Lo && "Invalid classification, type > 16 bytes.");
-          FieldEightBytes[0] = NoClass;
-          FieldEightBytes[1] = Integer;
-        } else {
-          FieldEightBytes[0] = Integer;
-          FieldEightBytes[1] = EB_Hi ? Integer : NoClass;
-        }
-      } else
-        classifyClang23(i->getType(), Offset, FieldEightBytes, isNamedArg,
-                        IsRegCall);
-      Lo = merge(Lo, FieldEightBytes[0]);
-      Hi = merge(Hi, FieldEightBytes[1]);
-      if (Lo == Memory || Hi == Memory)
-        break;
-    }
-
-    PostMerge(Size);
-  }
 }
 
 ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {
@@ -3251,7 +3180,8 @@ ABIArgInfo X86_64ABIInfo::classifyReturnType(QualType RetTy) const {
   // classification algorithm.
 
   SmallVector<X86_64ABIInfo::Class> EightBytes;
-  classify(RetTy, 0, EightBytes, /*isNamedArg*/ true);
+  classify(RetTy, 0, EightBytes, /*isNamedArg=*/true,
+           /*isRegCall=*/false, useLegacyClassificationAlgorithm());
 
   X86_64ABIInfo::Class Lo = NoClass;
   X86_64ABIInfo::Class Hi = NoClass;
@@ -3397,7 +3327,8 @@ X86_64ABIInfo::classifyArgumentType(QualType Ty, unsigned freeIntRegs,
   Ty = useFirstFieldIfTransparentUnion(Ty);
 
   SmallVector<X86_64ABIInfo::Class> EightBytes;
-  classify(Ty, 0, EightBytes, isNamedArg, IsRegCall);
+  classify(Ty, 0, EightBytes, isNamedArg, IsRegCall,
+           useLegacyClassificationAlgorithm() || IsRegCall);
 
   X86_64ABIInfo::Class Lo = NoClass;
   X86_64ABIInfo::Class Hi = NoClass;

>From 8b19574839e153e4b41ddc0aa1ab7e04f14b3d40 Mon Sep 17 00:00:00 2001
From: Benjamin Luke <benjamin.luke at sony.com>
Date: Thu, 1 Oct 2026 09:22:03 -0700
Subject: [PATCH 15/18] Add some additional coverage to make sure that the
 legacy clang23 classifer is unchanged

---
 clang/test/CodeGenCXX/x86_64-arguments.cpp | 47 +++++++++++++++++++++-
 1 file changed, 46 insertions(+), 1 deletion(-)

diff --git a/clang/test/CodeGenCXX/x86_64-arguments.cpp b/clang/test/CodeGenCXX/x86_64-arguments.cpp
index ebeba7fd654952..342e0d8593a2af 100644
--- a/clang/test/CodeGenCXX/x86_64-arguments.cpp
+++ b/clang/test/CodeGenCXX/x86_64-arguments.cpp
@@ -1,21 +1,27 @@
-// RUN: %clang_cc1 -no-enable-noundef-analysis -triple x86_64-unknown-unknown -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -no-enable-noundef-analysis -triple x86_64-unknown-unknown -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,OFFSET
+// RUN: %clang_cc1 -no-enable-noundef-analysis -triple x86_64-unknown-unknown -fclang-abi-compat=23 -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,OFFSET
+// RUN: %clang_cc1 -no-enable-noundef-analysis -triple x86_64-unknown-linux-gnux32 -emit-llvm -o - %s | FileCheck %s --check-prefixes=X32,OFFSET
+// RUN: %clang_cc1 -no-enable-noundef-analysis -triple x86_64-unknown-linux-gnux32 -fclang-abi-compat=23 -emit-llvm -o - %s | FileCheck %s --check-prefixes=X32,OFFSET
 
 // Basic base class test.
 struct f0_s0 { unsigned a; };
 struct f0_s1 : public f0_s0 { void *b; };
 // CHECK-LABEL: define{{.*}} void @_Z2f05f0_s1(i32 %a0.coerce0, ptr %a0.coerce1)
+// X32-LABEL: define{{.*}} void @_Z2f05f0_s1(i64 %a0.coerce)
 void f0(f0_s1 a0) { }
 
 // Check with two eight-bytes in base class.
 struct f1_s0 { unsigned a; unsigned b; float c; };
 struct f1_s1 : public f1_s0 { float d;};
 // CHECK-LABEL: define{{.*}} void @_Z2f15f1_s1(i64 %a0.coerce0, <2 x float> %a0.coerce1)
+// X32-LABEL: define{{.*}} void @_Z2f15f1_s1(i64 %a0.coerce0, <2 x float> %a0.coerce1)
 void f1(f1_s1 a0) { }
 
 // Check with two eight-bytes in base class and merge.
 struct f2_s0 { unsigned a; unsigned b; float c; };
 struct f2_s1 : public f2_s0 { char d;};
 // CHECK-LABEL: define{{.*}} void @_Z2f25f2_s1(i64 %a0.coerce0, i64 %a0.coerce1)
+// X32-LABEL: define{{.*}} void @_Z2f25f2_s1(i64 %a0.coerce0, i64 %a0.coerce1)
 void f2(f2_s1 a0) { }
 
 // PR5831
@@ -26,6 +32,8 @@ void f3(struct s3_1 x) {}
 
 // CHECK-LABEL: define{{.*}} i64 @_Z4f4_0M2s4i(i64 %a)
 // CHECK: define {{.*}} @_Z4f4_1M2s4FivE(i64 %a.coerce0, i64 %a.coerce1)
+// X32-LABEL: define{{.*}} i32 @_Z4f4_0M2s4i(i32 %a)
+// X32-LABEL: define{{.*}} i64 @_Z4f4_1M2s4FivE(i64 %a.coerce)
 struct s4 {};
 typedef int s4::* s4_mdp;
 typedef int (s4::*s4_mfp)();
@@ -35,16 +43,19 @@ s4_mfp f4_1(s4_mfp a) { return a; }
 // A struct with <= one eightbyte before a member data pointer should still
 // be allowed in registers.
 // CHECK-LABEL: define{{.*}} void @{{.*}}f_struct_with_mdp{{.*}}(ptr %a.coerce0, i64 %a.coerce1)
+// X32-LABEL: define{{.*}} void @_Z17f_struct_with_mdp15struct_with_mdp(i64 %a.coerce)
 struct struct_with_mdp { char *a; s4_mdp b; };
 void f_struct_with_mdp(struct_with_mdp a) { (void)a; }
 
 // A struct with anything before a member function will be too big and
 // goes in memory.
 // CHECK-LABEL: define{{.*}} void @{{.*}}f_struct_with_mfp_0{{.*}}(ptr byval(%struct{{.*}}) align 8 %a)
+// X32-LABEL: define{{.*}} void @_Z19f_struct_with_mfp_017struct_with_mfp_0(i64 %a.coerce0, i32 %a.coerce1)
 struct struct_with_mfp_0 { char a; s4_mfp b; };
 void f_struct_with_mfp_0(struct_with_mfp_0 a) { (void)a; }
 
 // CHECK-LABEL: define{{.*}} void @{{.*}}f_struct_with_mfp_1{{.*}}(ptr byval(%struct{{.*}}) align 8 %a)
+// X32-LABEL: define{{.*}} void @_Z19f_struct_with_mfp_117struct_with_mfp_1(i64 %a.coerce0, i32 %a.coerce1)
 struct struct_with_mfp_1 { void *a; s4_mfp b; };
 void f_struct_with_mfp_1(struct_with_mfp_1 a) { (void)a; }
 
@@ -218,3 +229,37 @@ union U {
 int f(union U u) { return u.f2[1]; }
 // CHECK-LABEL: define{{.*}} i32 @_ZN6test111fENS_1UE(i32
 }
+
+// A complex float starting at bit 32 has its real and imaginary components
+// in different eightbytes.
+struct ComplexAtBit32 {
+  float prefix;
+  _Complex float value;
+};
+// OFFSET-LABEL: define{{.*}} void @take_complex_at_bit32(<2 x float> %x.coerce0, float %x.coerce1)
+extern "C" void take_complex_at_bit32(ComplexAtBit32 x) {}
+// OFFSET-LABEL: define{{.*}} { <2 x float>, float } @return_complex_at_bit32(<2 x float> %x.coerce0, float %x.coerce1)
+extern "C" ComplexAtBit32 return_complex_at_bit32(ComplexAtBit32 x) {
+  return x;
+}
+
+// The third eightbyte makes this whole record pass and return in memory.
+struct ComplexAtBit96 {
+  unsigned char prefix[12];
+  _Complex float value;
+};
+// OFFSET-LABEL: define{{.*}} void @take_complex_at_bit96(ptr byval(%struct.ComplexAtBit96) align 8 %x)
+extern "C" void take_complex_at_bit96(ComplexAtBit96 x) {}
+// OFFSET-LABEL: define{{.*}} void @return_complex_at_bit96(ptr {{.*}}sret(%struct.ComplexAtBit96) align 4 %agg.result, ptr byval(%struct.ComplexAtBit96) align 8 %x)
+extern "C" ComplexAtBit96 return_complex_at_bit96(ComplexAtBit96 x) {
+  return x;
+}
+
+// The array elements begin at bits 32 and 64, so classification must merge
+// their classes into the appropriate eightbytes of the containing record.
+struct FloatArrayAtBit32 {
+  float prefix;
+  float values[2];
+};
+// OFFSET-LABEL: define{{.*}} void @take_float_array_at_bit32(<2 x float> %x.coerce0, float %x.coerce1)
+extern "C" void take_float_array_at_bit32(FloatArrayAtBit32 x) {}

>From 55e101bf051d3e784e5168d801b18e40806cef3b Mon Sep 17 00:00:00 2001
From: Benjamin Luke <benjamin.luke at sony.com>
Date: Mon, 2 Nov 2026 09:25:25 -0800
Subject: [PATCH 16/18] Classify matrix types as memory

---
 clang/lib/CodeGen/Targets/X86.cpp | 33 +++++++++++++++++++------------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index 54d0401cde3ec6..8dd423a862c1f8 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -1366,8 +1366,7 @@ class X86_64ABIInfo : public ABIInfo {
   /// Collect the canonical element type and shared classification properties
   /// of the vector \param VT or matrix \param MT. At least one must be
   /// non-null.
-  VectorTypeInfo getVectorTypeInfo(const VectorType *VT,
-                                   const MatrixType *MT = nullptr) const;
+  VectorTypeInfo getVectorTypeInfo(const VectorType *VT) const;
 
   /// Return true when \param AT must be passed in memory because of the SysV
   /// size limit or the alignment of its base. Regcall ignores the size limit.
@@ -2087,11 +2086,9 @@ X86_64ABIInfo::getClassPairForSpan(uint64_t Offset, uint64_t Size, Class C) {
 }
 
 X86_64ABIInfo::VectorTypeInfo
-X86_64ABIInfo::getVectorTypeInfo(const VectorType *VT,
-                                 const MatrixType *MT) const {
-  assert((VT || MT) && "expected a vector or matrix type");
-  QualType ElementType = getContext().getCanonicalType(
-      VT ? VT->getElementType() : MT->getElementType());
+X86_64ABIInfo::getVectorTypeInfo(const VectorType *VT) const {
+  assert((VT) && "expected a vector type");
+  QualType ElementType = getContext().getCanonicalType(VT->getElementType());
   bool IsFPElement = ElementType->isFloat16Type() ||
                      ElementType->isBFloat16Type() ||
                      ElementType == getContext().FloatTy ||
@@ -2511,18 +2508,15 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
     return;
   }
 
-  // Vector & Matrix types
-  // Matrix types are "flattened" and then classified as a m*n vector with the
-  // same element type.
+  // Vector types
   const VectorType *VT = Ty->getAs<VectorType>();
-  const MatrixType *MT = Ty->getAs<MatrixType>();
-  if (VT || (MT && !useLegacy)) {
+  if (VT) {
     if (useLegacy) {
       classifyLegacyVector(VT, OffsetBase, EightBytes, isNamedArg);
       return;
     }
 
-    VectorTypeInfo Info = getVectorTypeInfo(VT, MT);
+    VectorTypeInfo Info = getVectorTypeInfo(VT);
 
     auto SetEightByteClasses = [&](Class FirstClass, Class RestClass) {
       // Vector objects should not start partway through one eightbyte and
@@ -2614,6 +2608,19 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
     return;
   }
 
+  const ConstantMatrixType *MT = Ty->getAs<ConstantMatrixType>();
+  if (MT) {
+    if (useLegacy)
+      return;
+
+    // FIXME: Matrices use array-like size and alignment. Applying vector rules can produce invalid 
+    // SSE/SSEUP classification. Classifying as array exposes problems with sub byte elements like 
+    // _BitInt(4) since array memory layout will pad to one byte per elm but vector loads and stores 
+    // will assume packed elm with no padding. For now just classify as memory.
+    ClassifyAsMemory();
+    return;
+  }
+
   // Arrays
   if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
     if (useLegacy) {

>From 5b7e17f94978086fd7113d1658b0e265e5ffdda5 Mon Sep 17 00:00:00 2001
From: Benjamin Luke <benjamin.luke at sony.com>
Date: Mon, 2 Nov 2026 09:45:23 -0800
Subject: [PATCH 17/18] clang format

---
 clang/lib/CodeGen/Targets/X86.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index 8dd423a862c1f8..1c1c8444659d79 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -2613,9 +2613,10 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
     if (useLegacy)
       return;
 
-    // FIXME: Matrices use array-like size and alignment. Applying vector rules can produce invalid 
-    // SSE/SSEUP classification. Classifying as array exposes problems with sub byte elements like 
-    // _BitInt(4) since array memory layout will pad to one byte per elm but vector loads and stores 
+    // FIXME: Matrices use array-like size and alignment. Applying vector rules
+    // can produce invalid SSE/SSEUP classification. Classifying as array
+    // exposes problems with sub byte elements like _BitInt(4) since array
+    // memory layout will pad to one byte per elm but vector loads and stores
     // will assume packed elm with no padding. For now just classify as memory.
     ClassifyAsMemory();
     return;

>From a0ef7fe89c5068521a5857fa38883598c0a57021 Mon Sep 17 00:00:00 2001
From: Benjamin Luke <benjamin.luke at sony.com>
Date: Tue, 3 Nov 2026 16:46:18 -0800
Subject: [PATCH 18/18] Fix tests after rebase

---
 clang/lib/CodeGen/Targets/X86.cpp             |  3 +-
 .../call-conv-lowering-x86_64-abi-size.c      | 11 +++--
 .../CodeGen/call-conv-lowering-x86_64-avx.c   | 21 +++++----
 .../call-conv-lowering-x86_64-npot-vector.c   | 44 +++++++++++++------
 ...-conv-lowering-x86_64-union-tail-padding.c | 34 ++++++++------
 .../call-conv-lowering-x86_64-x87-vector.c    | 39 ++++++++++------
 clang/test/CodeGen/X86/x86_64-abi-size.c      | 12 +++--
 .../test/CodeGen/X86/x86_64-vector-abi-size.c | 17 ++++---
 8 files changed, 117 insertions(+), 64 deletions(-)

diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index 1c1c8444659d79..3b4accc5766bd1 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -3009,7 +3009,8 @@ llvm::Type *X86_64ABIInfo::GetSSETypeAtOffset(llvm::Type *IRType,
   // Keep 16-bit FP vector lanes visible when they fit in one SSE eightbyte.
   // Reinterpreting them as double would make attributes like nofpclass describe
   // carrier bits rather than the source vector elements.
-  if (IROffset == 0 && SourceOffset == 0) {
+  if (!useLegacyClassificationAlgorithm() && IROffset == 0 &&
+      SourceOffset == 0) {
     if (auto *VTy = dyn_cast<llvm::FixedVectorType>(IRType)) {
       llvm::Type *EltTy = VTy->getElementType();
       if (EltTy->is16bitFPTy() && TD.getTypeAllocSize(VTy) <= 8)
diff --git a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-abi-size.c b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-abi-size.c
index b38fa3a7b27a63..1e2f340a22356b 100644
--- a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-abi-size.c
+++ b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-abi-size.c
@@ -1,9 +1,11 @@
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
 // RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
-// RUN: FileCheck --check-prefix=LLVM --input-file=%t-cir.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,DIRECT --input-file=%t-cir.ll %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
-// RUN: FileCheck --check-prefix=LLVM --input-file=%t.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,DEFAULT --input-file=%t.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclang-abi-compat=23 -emit-llvm %s -o %t-compat23.ll
+// RUN: FileCheck --check-prefixes=LLVM,DIRECT --input-file=%t-compat23.ll %s
 
 // CIR-DAG: ![[U64U64:rec_anon_struct[0-9]*]] = !cir.struct<{data !u64i, data !u64i}>
 // CIR-DAG: ![[S64U8:rec_anon_struct[0-9]*]] = !cir.struct<{data !s64i, data !u8i}>
@@ -120,8 +122,11 @@ struct LongC1 longc1(struct LongC1 s) { return s; }
 // LLVM: define dso_local { i64, i64 } @longc1(i64 %{{.+}}, i64 %{{.+}})
 
 typedef float f1 __attribute__((ext_vector_type(1)));
+// Default CodeGen passes a single-element float vector and its wrapper in
+// memory. CIR and Clang 23 pass direct.
 struct LongF1 { long l; f1 v; };
 struct LongF1 longf1(struct LongF1 s) { return s; }
 
 // CIR: cir.func {{.*}}@longf1(%arg0: !s64i loc({{[^)]+}}), %arg1: !u64i loc({{[^)]+}})) -> ![[S64U64]] attributes
-// LLVM: define dso_local { i64, i64 } @longf1(i64 %{{.+}}, i64 %{{.+}})
+// DIRECT: define dso_local { i64, i64 } @longf1(i64 %{{.+}}, i64 %{{.+}})
+// DEFAULT: define dso_local void @longf1(ptr dead_on_unwind noalias writable sret(%struct.LongF1) align 8 %{{[^,)]+}}, ptr noundef byval(%struct.LongF1) align 8 %{{[^,)]+}})
diff --git a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-avx.c b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-avx.c
index bd9537a4296e5e..9357a33789157f 100644
--- a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-avx.c
+++ b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-avx.c
@@ -1,23 +1,25 @@
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
 // RUN: FileCheck --check-prefixes=CIR,CIR-SSE --input-file=%t.cir %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
-// RUN: FileCheck --check-prefixes=LLVM,LLVM-SSE,LLVM-CIR-SSE --input-file=%t-cir.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-SSE,LLVM-CIR-SSE,LLVM-DIRECT-RETURN --input-file=%t-cir.ll %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
-// RUN: FileCheck --check-prefixes=LLVM,LLVM-SSE,LLVM-OGCG-SSE --input-file=%t.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-SSE,LLVM-OGCG-SSE,LLVM-MEMORY-RETURN --input-file=%t.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclang-abi-compat=23 -emit-llvm %s -o %t-compat23-return.ll
+// RUN: FileCheck --check-prefix=LLVM-DIRECT-RETURN --input-file=%t-compat23-return.ll %s
 
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx -fclangir -emit-cir %s -o %t-avx.cir
 // RUN: FileCheck --check-prefixes=CIR,CIR-AVX --input-file=%t-avx.cir %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx -fclangir -emit-llvm %s -o %t-avx-cir.ll
-// RUN: FileCheck --check-prefixes=LLVM,LLVM-AVX,LLVM-CIR-AVX --input-file=%t-avx-cir.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-AVX,LLVM-CIR-AVX,LLVM-DIRECT-RETURN --input-file=%t-avx-cir.ll %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx -emit-llvm %s -o %t-avx.ll
-// RUN: FileCheck --check-prefixes=LLVM,LLVM-AVX,LLVM-OGCG-AVX --input-file=%t-avx.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-AVX,LLVM-OGCG-AVX,LLVM-DIRECT-RETURN --input-file=%t-avx.ll %s
 
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx512f -fclangir -emit-cir %s -o %t-avx512.cir
 // RUN: FileCheck --check-prefix=CIR-AVX512 --input-file=%t-avx512.cir %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx512f -fclangir -emit-llvm %s -o %t-avx512-cir.ll
-// RUN: FileCheck --check-prefixes=LLVM,LLVM-AVX512 --input-file=%t-avx512-cir.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-AVX512,LLVM-DIRECT-RETURN --input-file=%t-avx512-cir.ll %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx512f -emit-llvm %s -o %t-avx512.ll
-// RUN: FileCheck --check-prefixes=LLVM,LLVM-AVX512 --input-file=%t-avx512.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-AVX512,LLVM-DIRECT-RETURN --input-file=%t-avx512.ll %s
 
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclang-abi-compat=23 -fclangir -emit-llvm %s -o %t-compat23-cir.ll
 // RUN: FileCheck --check-prefix=LLVM-CIR-PINNED --input-file=%t-compat23-cir.ll %s
@@ -55,11 +57,12 @@ void take_v256(v8f v) { (void)v; }
 // LLVM-AVX: define dso_local void @take_v256(<8 x float> noundef %{{[^,)]+}})
 // LLVM-AVX512: define dso_local void @take_v256(<8 x float> noundef %{{[^,)]+}})
 
-// The register-versus-memory split applies to arguments only.  A 256-bit
-// vector comes back in registers at every level.
+// Without AVX, the clang24 classifier returns a 256-bit vector in memory since it is wider than 128bits.
+// CIR and Clang 23 retain the direct return.
 v8f ret_v256(void) { v8f z = {0}; return z; }
 
-// LLVM: define dso_local <8 x float> @ret_v256()
+// LLVM-DIRECT-RETURN: define dso_local <8 x float> @ret_v256()
+// LLVM-MEMORY-RETURN: define dso_local void @ret_v256(ptr dead_on_unwind noalias writable sret(<8 x float>) align 32 %{{[^,)]+}})
 
 // Below AVX the caller has to build the byval slot, which is the memory copy a
 // vector argument needs on a non-variadic call.
diff --git a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-npot-vector.c b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-npot-vector.c
index 101384092fb9fd..296826d858678a 100644
--- a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-npot-vector.c
+++ b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-npot-vector.c
@@ -3,21 +3,29 @@
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx -fclangir -emit-cir %s -o %t-avx.cir
 // RUN: FileCheck --check-prefixes=CIR,CIR-AVX --input-file=%t-avx.cir %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
-// RUN: FileCheck --check-prefixes=LLVM,LLVMCIR,NOAVX,NOAVX512 --input-file=%t-cir.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,LLVMCIR,NOAVX,NOAVX512,WIDE-DIRECT-NOAVX,HALF-DOUBLE,WIDE-DIRECT --input-file=%t-cir.ll %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
-// RUN: FileCheck --check-prefixes=LLVM,OGCG,NOAVX,NOAVX512 --input-file=%t.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,OGCG,NOAVX,NOAVX512,HALF-VECTOR,WIDE-MEMORY --input-file=%t.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclang-abi-compat=23 -emit-llvm %s -o %t-compat23.ll
+// RUN: FileCheck --check-prefixes=LLVM,OGCG,NOAVX,NOAVX512,WIDE-DIRECT-NOAVX,HALF-DOUBLE,WIDE-DIRECT --input-file=%t-compat23.ll %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx -fclangir -emit-llvm %s -o %t-cir-avx.ll
-// RUN: FileCheck --check-prefixes=LLVM,LLVMCIR,AVX,NOAVX512 --input-file=%t-cir-avx.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,LLVMCIR,AVX,NOAVX512,HALF-DOUBLE,WIDE-DIRECT --input-file=%t-cir-avx.ll %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx -emit-llvm %s -o %t-avx.ll
-// RUN: FileCheck --check-prefixes=LLVM,OGCG,AVX,NOAVX512 --input-file=%t-avx.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,OGCG,AVX,NOAVX512,HALF-VECTOR,WIDE-DIRECT --input-file=%t-avx.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx -fclang-abi-compat=23 -emit-llvm %s -o %t-compat23-avx.ll
+// RUN: FileCheck --check-prefixes=LLVM,OGCG,AVX,NOAVX512,HALF-DOUBLE,WIDE-DIRECT --input-file=%t-compat23-avx.ll %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx512f -fclangir -emit-llvm %s -o %t-cir-avx512.ll
-// RUN: FileCheck --check-prefixes=LLVM,LLVMCIR,AVX,AVX512 --input-file=%t-cir-avx512.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,LLVMCIR,AVX,AVX512,HALF-DOUBLE,WIDE-DIRECT --input-file=%t-cir-avx512.ll %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx512f -emit-llvm %s -o %t-avx512.ll
-// RUN: FileCheck --check-prefixes=LLVM,OGCG,AVX,AVX512 --input-file=%t-avx512.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,OGCG,AVX,AVX512,HALF-VECTOR,WIDE-DIRECT --input-file=%t-avx512.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx512f -fclang-abi-compat=23 -emit-llvm %s -o %t-compat23-avx512.ll
+// RUN: FileCheck --check-prefixes=LLVM,OGCG,AVX,AVX512,HALF-DOUBLE,WIDE-DIRECT --input-file=%t-compat23-avx512.ll %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -menable-no-nans -menable-no-infs -fclangir -emit-llvm %s -o %t-cir-finite.ll
-// RUN: FileCheck --check-prefix=FINITE --input-file=%t-cir-finite.ll %s
+// RUN: FileCheck --check-prefixes=FINITE,FINITE-DOUBLE --input-file=%t-cir-finite.ll %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -menable-no-nans -menable-no-infs -emit-llvm %s -o %t-finite.ll
-// RUN: FileCheck --check-prefix=FINITE --input-file=%t-finite.ll %s
+// RUN: FileCheck --check-prefixes=FINITE,FINITE-VECTOR --input-file=%t-finite.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -menable-no-nans -menable-no-infs -fclang-abi-compat=23 -emit-llvm %s -o %t-compat23-finite.ll
+// RUN: FileCheck --check-prefixes=FINITE,FINITE-DOUBLE --input-file=%t-compat23-finite.ll %s
 
 typedef char c3 __attribute__((ext_vector_type(3)));
 typedef char c4 __attribute__((ext_vector_type(4)));
@@ -176,8 +184,12 @@ void take_h3(h3 v) {}
 // LLVM-LABEL: define dso_local void @take_uf(<2 x double> %{{.+}})
 // LLVM-LABEL: define dso_local void @take_uff(double %{{.+}}, double %{{.+}})
 // LLVM-LABEL: define dso_local void @take_ss3s(double %{{.+}}, i16 %{{.+}})
-// LLVM-LABEL: define dso_local void @take_h3(double %{{.+}})
-// FINITE-LABEL: define dso_local void @take_h3(double nofpclass(nan inf) %{{.+}})
+// Default CodeGen preserves the half floats so nofpclass applies to each.
+// CIR and Clang 23 retain the historical double.
+// HALF-DOUBLE-LABEL: define dso_local void @take_h3(double %{{.+}})
+// HALF-VECTOR-LABEL: define dso_local void @take_h3(<3 x half> noundef %{{.+}})
+// FINITE-DOUBLE-LABEL: define dso_local void @take_h3(double nofpclass(nan inf) %{{.+}})
+// FINITE-VECTOR-LABEL: define dso_local void @take_h3(<3 x half> noundef nofpclass(nan inf) %{{.+}})
 
 // Each half of a flattened argument carries the argument's attributes.
 void take_cd(_Complex double c) {}
@@ -205,14 +217,17 @@ i6 pass_i6(i6 v) { return v; }
 // CIR-AVX: cir.func{{.*}} @pass_f5(%arg0: !cir.vector<5 x !cir.float> {llvm.noundef} loc({{.+}})) -> !cir.vector<5 x !cir.float>
 // CIR-AVX: cir.func{{.*}} @pass_i6(%arg0: !cir.vector<6 x !s32i> {llvm.noundef} loc({{.+}})) -> !cir.vector<6 x !s32i>
 // NOAVX-LABEL: define dso_local void @take_d3(ptr noundef byval(<3 x double>) align 32 %{{.+}})
-// NOAVX-LABEL: define dso_local <5 x float> @pass_f5(ptr noundef byval(<5 x float>) align 32 %{{.+}})
-// NOAVX-LABEL: define dso_local <6 x i32> @pass_i6(ptr noundef byval(<6 x i32>) align 32 %{{.+}})
+// WIDE-DIRECT-NOAVX-LABEL: define dso_local <5 x float> @pass_f5(ptr noundef byval(<5 x float>) align 32 %{{.+}})
+// WIDE-MEMORY-LABEL: define dso_local void @pass_f5(ptr dead_on_unwind noalias writable sret(<5 x float>) align 32 %{{.+}}, ptr noundef byval(<5 x float>) align 32 %{{.+}})
+// WIDE-DIRECT-NOAVX-LABEL: define dso_local <6 x i32> @pass_i6(ptr noundef byval(<6 x i32>) align 32 %{{.+}})
+// WIDE-MEMORY-LABEL: define dso_local void @pass_i6(ptr dead_on_unwind noalias writable sret(<6 x i32>) align 32 %{{.+}}, ptr noundef byval(<6 x i32>) align 32 %{{.+}})
 // AVX-LABEL: define dso_local void @take_d3(<3 x double> noundef %{{.+}})
 // AVX-LABEL: define dso_local <5 x float> @pass_f5(<5 x float> noundef %{{.+}})
 // AVX-LABEL: define dso_local <6 x i32> @pass_i6(<6 x i32> noundef %{{.+}})
 // A byval pointer does not carry the vector's nofpclass.
 // FINITE-LABEL: define dso_local void @take_d3(ptr noundef byval(<3 x double>) align 32 %{{.+}})
-// FINITE-LABEL: define dso_local nofpclass(nan inf) <5 x float> @pass_f5(ptr noundef byval(<5 x float>) align 32 %{{.+}})
+// FINITE-DOUBLE-LABEL: define dso_local nofpclass(nan inf) <5 x float> @pass_f5(ptr noundef byval(<5 x float>) align 32 %{{.+}})
+// FINITE-VECTOR-LABEL: define dso_local void @pass_f5(ptr dead_on_unwind noalias writable sret(<5 x float>) align 32 %{{.+}}, ptr noundef byval(<5 x float>) align 32 %{{.+}})
 
 struct SF5 { f5 v; };
 struct SF5 pass_sf5(struct SF5 s) { return s; }
@@ -260,7 +275,8 @@ f5 va_f5(int n, ...) {
 // CIR-LABEL: cir.func{{.*}} @va_f5(
 // CIR: %[[STRIDE:[0-9]+]] = cir.const #cir.int<32> : !s32i
 // CIR-NEXT: cir.ptr_stride %{{[0-9]+}}, %[[STRIDE]] : (!cir.ptr<!u8i>, !s32i) -> !cir.ptr<!u8i>
-// LLVM-LABEL: define dso_local <5 x float> @va_f5(
+// WIDE-DIRECT-LABEL: define dso_local <5 x float> @va_f5(
+// WIDE-MEMORY-LABEL: define dso_local void @va_f5(ptr dead_on_unwind noalias writable sret(<5 x float>) align 32 %{{[^,)]+}},
 // LLVM: call ptr @llvm.ptrmask.p0.i64(ptr %{{.+}}, i64 -32)
 // LLVMCIR: getelementptr i8, ptr %{{.+}}, i64 32
 // OGCG: getelementptr i8, ptr %{{.+}}, i32 32
diff --git a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-union-tail-padding.c b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-union-tail-padding.c
index 8573e1e8527b24..daba4c345768a4 100644
--- a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-union-tail-padding.c
+++ b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-union-tail-padding.c
@@ -1,15 +1,19 @@
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
 // RUN: FileCheck --check-prefixes=CIR,CIR-SSE --input-file=%t.cir %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
-// RUN: FileCheck --check-prefixes=LLVM,LLVM-SSE --input-file=%t-cir.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-MEMORY --input-file=%t-cir.ll %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
-// RUN: FileCheck --check-prefixes=LLVM,LLVM-SSE --input-file=%t.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-MEMORY --input-file=%t.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclang-abi-compat=23 -emit-llvm %s -o %t-compat23.ll
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-MEMORY --input-file=%t-compat23.ll %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx512f -fclangir -emit-cir %s -o %t-avx.cir
 // RUN: FileCheck --check-prefixes=CIR,CIR-AVX --input-file=%t-avx.cir %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx512f -fclangir -emit-llvm %s -o %t-avx-cir.ll
-// RUN: FileCheck --check-prefixes=LLVM,LLVM-AVX --input-file=%t-avx-cir.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-DIRECT --input-file=%t-avx-cir.ll %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx512f -emit-llvm %s -o %t-avx.ll
-// RUN: FileCheck --check-prefixes=LLVM,LLVM-AVX --input-file=%t-avx.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-MEMORY --input-file=%t-avx.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx512f -fclang-abi-compat=23 -emit-llvm %s -o %t-avx-compat23.ll
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-DIRECT --input-file=%t-avx-compat23.ll %s
 
 typedef float v4f __attribute__((vector_size(16)));
 typedef float v2f __attribute__((vector_size(8)));
@@ -108,20 +112,21 @@ void take_over_aligned32(OverAligned32 u) {}
 // CIR: cir.func{{.*}} @take_over_aligned32(%arg0: !cir.ptr<!rec_OverAligned32> {llvm.align = 32 : i64, llvm.byval = !rec_OverAligned32, llvm.noundef} loc{{.*}})
 // LLVM: define{{.*}} void @take_over_aligned32(ptr noundef byval(%union.OverAligned32) align 32 %{{.+}})
 
-// 32 bytes against members of 16 and 17.  The quad reaches SSEUP, so with AVX
-// the coerce is named from the union's size, and without it memory.
+// The default classifier tracks all four eightbytes, so the char array makes
+// this union MEMORY. CIR and Clang 23 classify as SSE SSEUP with AVX-512.
 void take_quad32(Quad32 u) {}
 // CIR-SSE: cir.func{{.*}} @take_quad32(%arg0: !cir.ptr<!rec_Quad32> {llvm.align = 16 : i64, llvm.byval = !rec_Quad32, llvm.noundef} loc{{.*}})
 // CIR-AVX: cir.func{{.*}} @take_quad32(%arg0: !cir.vector<4 x !cir.double> loc{{.*}})
-// LLVM-SSE: define{{.*}} void @take_quad32(ptr noundef byval(%union.Quad32) align 16 %{{.+}})
-// LLVM-AVX: define{{.*}} void @take_quad32(<4 x double> %{{.+}})
+// LLVM-MEMORY: define{{.*}} void @take_quad32(ptr noundef byval(%union.Quad32) align 16 %{{.+}})
+// LLVM-DIRECT: define{{.*}} void @take_quad32(<4 x double> %{{.+}})
 
-// 64 bytes, the widest size an SSEUP coerce can be named from.
+// The same distinction applies at 64 bytes: default CodeGen uses MEMORY,
+// while CIR and Clang 23 retain classify as SSE SSEUP with AVX-512.
 void take_quad64(Quad64 u) {}
 // CIR-SSE: cir.func{{.*}} @take_quad64(%arg0: !cir.ptr<!rec_Quad64> {llvm.align = 16 : i64, llvm.byval = !rec_Quad64, llvm.noundef} loc{{.*}})
 // CIR-AVX: cir.func{{.*}} @take_quad64(%arg0: !cir.vector<8 x !cir.double> loc{{.*}})
-// LLVM-SSE: define{{.*}} void @take_quad64(ptr noundef byval(%union.Quad64) align 16 %{{.+}})
-// LLVM-AVX: define{{.*}} void @take_quad64(<8 x double> %{{.+}})
+// LLVM-MEMORY: define{{.*}} void @take_quad64(ptr noundef byval(%union.Quad64) align 16 %{{.+}})
+// LLVM-DIRECT: define{{.*}} void @take_quad64(<8 x double> %{{.+}})
 
 // 80 bytes, past 512, so the quad member cannot put it in registers.
 void take_quad80(Quad80 u) {}
@@ -133,12 +138,13 @@ void take_cplx_quad48(CplxQuad48 u) {}
 // CIR: cir.func{{.*}} @take_cplx_quad48(%arg0: !cir.ptr<!rec_CplxQuad48> {llvm.align = 16 : i64, llvm.byval = !rec_CplxQuad48, llvm.noundef} loc{{.*}})
 // LLVM: define{{.*}} void @take_cplx_quad48(ptr noundef byval(%union.CplxQuad48) align 16 %{{.+}})
 
-// A vector reaches SSEUP too, here with nothing spanning the union.
+// A vector member also produces the legacy SSE SSEUP classification with AVX-512,
+// while default/clang24 CodeGen classifies the char array and passes the union in memory.
 void take_vec_tail_pad(VecTailPad u) {}
 // CIR-SSE: cir.func{{.*}} @take_vec_tail_pad(%arg0: !cir.ptr<!rec_VecTailPad> {llvm.align = 16 : i64, llvm.byval = !rec_VecTailPad, llvm.noundef} loc{{.*}})
 // CIR-AVX: cir.func{{.*}} @take_vec_tail_pad(%arg0: !cir.vector<4 x !cir.double> loc{{.*}})
-// LLVM-SSE: define{{.*}} void @take_vec_tail_pad(ptr noundef byval(%union.VecTailPad) align 16 %{{.+}})
-// LLVM-AVX: define{{.*}} void @take_vec_tail_pad(<4 x double> %{{.+}})
+// LLVM-MEMORY: define{{.*}} void @take_vec_tail_pad(ptr noundef byval(%union.VecTailPad) align 16 %{{.+}})
+// LLVM-DIRECT: define{{.*}} void @take_vec_tail_pad(<4 x double> %{{.+}})
 
 // A vector narrower than 128 bits never reaches SSEUP, so this union is
 // classified from its size at every target.
diff --git a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-x87-vector.c b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-x87-vector.c
index 52f8f6a705a82e..47c59d74793239 100644
--- a/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-x87-vector.c
+++ b/clang/test/CIR/CodeGen/call-conv-lowering-x86_64-x87-vector.c
@@ -1,23 +1,29 @@
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-psabi -fclangir -emit-cir %s -o %t.cir
 // RUN: FileCheck --check-prefixes=CIR,CIR-SSE --input-file=%t.cir %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-psabi -fclangir -emit-llvm %s -o %t-cir.ll
-// RUN: FileCheck --check-prefixes=LLVM,LLVM-SSE --input-file=%t-cir.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-SSE,LLVM-LD2-DIRECT,LLVM-LD3-DIRECT --input-file=%t-cir.ll %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-psabi -emit-llvm %s -o %t.ll
-// RUN: FileCheck --check-prefixes=LLVM,LLVM-SSE --input-file=%t.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-SSE,LLVM-LD2-MEMORY,LLVM-LD3-MEMORY --input-file=%t.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-psabi -fclang-abi-compat=23 -emit-llvm %s -o %t-compat23-sse.ll
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-SSE,LLVM-LD2-DIRECT,LLVM-LD3-DIRECT --input-file=%t-compat23-sse.ll %s
 
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-psabi -target-feature +avx -fclangir -emit-cir %s -o %t-avx.cir
 // RUN: FileCheck --check-prefixes=CIR,CIR-AVX --input-file=%t-avx.cir %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-psabi -target-feature +avx -fclangir -emit-llvm %s -o %t-avx-cir.ll
-// RUN: FileCheck --check-prefixes=LLVM,LLVM-AVX --input-file=%t-avx-cir.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-AVX,LLVM-LD2-DIRECT,LLVM-LD3-DIRECT --input-file=%t-avx-cir.ll %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-psabi -target-feature +avx -emit-llvm %s -o %t-avx.ll
-// RUN: FileCheck --check-prefixes=LLVM,LLVM-AVX --input-file=%t-avx.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-AVX,LLVM-LD2-DIRECT,LLVM-LD3-MEMORY --input-file=%t-avx.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-psabi -target-feature +avx -fclang-abi-compat=23 -emit-llvm %s -o %t-compat23-avx.ll
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-AVX,LLVM-LD2-DIRECT,LLVM-LD3-DIRECT --input-file=%t-compat23-avx.ll %s
 
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-psabi -target-feature +avx512f -fclangir -emit-cir %s -o %t-avx512.cir
 // RUN: FileCheck --check-prefixes=CIR,CIR-AVX512 --input-file=%t-avx512.cir %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-psabi -target-feature +avx512f -fclangir -emit-llvm %s -o %t-avx512-cir.ll
-// RUN: FileCheck --check-prefixes=LLVM,LLVM-AVX512 --input-file=%t-avx512-cir.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-AVX512,LLVM-LD2-DIRECT,LLVM-LD3-DIRECT --input-file=%t-avx512-cir.ll %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-psabi -target-feature +avx512f -emit-llvm %s -o %t-avx512.ll
-// RUN: FileCheck --check-prefixes=LLVM,LLVM-AVX512 --input-file=%t-avx512.ll %s
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-AVX512,LLVM-LD2-DIRECT,LLVM-LD3-DIRECT --input-file=%t-avx512.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-psabi -target-feature +avx512f -fclang-abi-compat=23 -emit-llvm %s -o %t-compat23-avx512.ll
+// RUN: FileCheck --check-prefixes=LLVM,LLVM-AVX512,LLVM-LD2-DIRECT,LLVM-LD3-DIRECT --input-file=%t-compat23-avx512.ll %s
 
 typedef long double ld1 __attribute__((vector_size(16)));
 typedef long double ld2 __attribute__((vector_size(32)));
@@ -154,14 +160,15 @@ ld2 arg_ld2(int n, ...) {
 // CIR: %[[VPTR:.+]] = cir.cast bitcast %[[ALIGNED]] : !cir.ptr<!u8i> -> !cir.ptr<!cir.vector<2 x !cir.long_double<!cir.f80>>>
 // CIR: cir.load align(32) %[[VPTR]] : !cir.ptr<!cir.vector<2 x !cir.long_double<!cir.f80>>>, !cir.vector<2 x !cir.long_double<!cir.f80>>
 
-// LLVM-LABEL: define dso_local <2 x x86_fp80> @arg_ld2(i32 noundef %{{[^,)]+}}, ...)
+// LLVM-LD2-DIRECT-LABEL: define dso_local <2 x x86_fp80> @arg_ld2(i32 noundef %{{[^,)]+}}, ...)
+// LLVM-LD2-MEMORY-LABEL: define dso_local void @arg_ld2(ptr dead_on_unwind noalias writable sret(<2 x x86_fp80>) align 32 %{{[^,)]+}}, i32 noundef %{{[^,)]+}}, ...)
 // LLVM-NOT: icmp
 // LLVM: %[[ALIGNED:.+]] = call ptr @llvm.ptrmask.p0.i64(ptr %{{.+}}, i64 -32)
 // LLVM: load <2 x x86_fp80>, ptr %[[ALIGNED]], align 32
 
-// Two elements take 32 bytes, which pass in memory below AVX.  The vector on
-// its own is still returned directly, but a record holding it is returned
-// through sret.
+// Two elements take 32 bytes, which pass in memory below AVX. The default
+// classifier also returns the vector through sret below AVX; CIR and Clang 23
+// retain the direct vector return. A record holding it uses sret below AVX.
 void take_ld2(ld2 v) {}
 
 // CIR-SSE: cir.func {{.*}}@take_ld2(%arg0: !cir.ptr<!cir.vector<2 x !cir.long_double<!cir.f80>>> {llvm.align = 32 : i64, llvm.byval = !cir.vector<2 x !cir.long_double<!cir.f80>>, llvm.noundef} loc({{[^)]+}}))
@@ -183,12 +190,14 @@ void call_ld2(ld2 v) { take_ld2(v); }
 ld2 ret_ld2(ld2 *p) { return *p; }
 
 // CIR: cir.func {{.*}}@ret_ld2(%arg0: !cir.ptr<!cir.vector<2 x !cir.long_double<!cir.f80>>> {llvm.noundef} loc({{[^)]+}})) -> !cir.vector<2 x !cir.long_double<!cir.f80>>
-// LLVM: define dso_local <2 x x86_fp80> @ret_ld2(ptr noundef %{{[^,)]+}})
+// LLVM-LD2-DIRECT: define dso_local <2 x x86_fp80> @ret_ld2(ptr noundef %{{[^,)]+}})
+// LLVM-LD2-MEMORY: define dso_local void @ret_ld2(ptr dead_on_unwind noalias writable sret(<2 x x86_fp80>) align 32 %{{[^,)]+}}, ptr noundef %{{[^,)]+}})
 
 ld2 call_ret_ld2(ld2 *p) { return ret_ld2(p); }
 
 // CIR: cir.call @ret_ld2(%{{.+}}) : (!cir.ptr<!cir.vector<2 x !cir.long_double<!cir.f80>>> {llvm.noundef}) -> !cir.vector<2 x !cir.long_double<!cir.f80>>
-// LLVM: call <2 x x86_fp80> @ret_ld2(ptr noundef %{{[^,)]+}})
+// LLVM-LD2-DIRECT: call <2 x x86_fp80> @ret_ld2(ptr noundef %{{[^,)]+}})
+// LLVM-LD2-MEMORY: call void @ret_ld2(ptr dead_on_unwind writable sret(<2 x x86_fp80>) align 32 %{{[^,)]+}}, ptr noundef %{{[^,)]+}})
 
 struct S2 { ld2 v; };
 struct S2 take_s2(struct S2 s) { return s; }
@@ -272,8 +281,10 @@ ld3 fwd_ld3(void) { return src_ld3(); }
 
 // CIR: cir.func {{.*}}@fwd_ld3() -> !cir.vector<3 x !cir.long_double<!cir.f80>>
 // CIR: cir.call @src_ld3() : () -> !cir.vector<3 x !cir.long_double<!cir.f80>>
-// LLVM: define dso_local <3 x x86_fp80> @fwd_ld3()
-// LLVM: call <3 x x86_fp80> @src_ld3()
+// LLVM-LD3-DIRECT: define dso_local <3 x x86_fp80> @fwd_ld3()
+// LLVM-LD3-MEMORY: define dso_local void @fwd_ld3(ptr dead_on_unwind noalias writable sret(<3 x x86_fp80>) align 64 %{{[^,)]+}})
+// LLVM-LD3-DIRECT: call <3 x x86_fp80> @src_ld3()
+// LLVM-LD3-MEMORY: call void @src_ld3(ptr dead_on_unwind writable sret(<3 x x86_fp80>) align 64 %{{[^,)]+}})
 
 void pass_ld3(void) { sink_ld3(src_ld3()); }
 
diff --git a/clang/test/CodeGen/X86/x86_64-abi-size.c b/clang/test/CodeGen/X86/x86_64-abi-size.c
index 2b44c1be48dfa2..19ee495ddc2435 100644
--- a/clang/test/CodeGen/X86/x86_64-abi-size.c
+++ b/clang/test/CodeGen/X86/x86_64-abi-size.c
@@ -1,5 +1,6 @@
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o - | FileCheck %s
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fexperimental-abi-lowering %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,DEFAULT
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fclang-abi-compat=23 %s -o - | FileCheck %s --check-prefixes=CHECK,DIRECT
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fexperimental-abi-lowering %s -o - | FileCheck %s --check-prefixes=CHECK,DIRECT
 
 typedef _Bool b3 __attribute__((ext_vector_type(3)));
 typedef _Bool b4 __attribute__((ext_vector_type(4)));
@@ -180,7 +181,7 @@ struct LongB12 longb12(struct LongB12 s) { return s; }
 ub1x4 vub1x4(ub1x4 v) { return v; }
 // CHECK: define dso_local i32 @vub1x4(i32 %{{.*}})
 
-// A one-bit _BitInt, char or float vector is not a bool vector, so the
+// A one-bit _BitInt or char vector is not a bool vector, so the
 // eightbyte holding it stays an i64.
 struct LongUB1x4 { long l; ub1x4 v; };
 struct LongUB1x4 longub1x4(struct LongUB1x4 s) { return s; }
@@ -194,6 +195,9 @@ struct LongC1 { long l; c1 v; };
 struct LongC1 longc1(struct LongC1 s) { return s; }
 // CHECK: define dso_local { i64, i64 } @longc1(i64 %{{.*}}, i64 %{{.*}})
 
+// The default classifier passes a single-element float vector and its wrapper
+// in memory. Clang 23 and experimental ABI lowering retain the direct coercion.
 struct LongF1 { long l; f1 v; };
 struct LongF1 longf1(struct LongF1 s) { return s; }
-// CHECK: define dso_local { i64, i64 } @longf1(i64 %{{.*}}, i64 %{{.*}})
+// DEFAULT: define dso_local void @longf1(ptr dead_on_unwind noalias writable sret(%struct.LongF1) align 8 %{{.*}}, ptr noundef byval(%struct.LongF1) align 8 %{{.*}})
+// DIRECT: define dso_local { i64, i64 } @longf1(i64 %{{.*}}, i64 %{{.*}})
diff --git a/clang/test/CodeGen/X86/x86_64-vector-abi-size.c b/clang/test/CodeGen/X86/x86_64-vector-abi-size.c
index a38513dfb70431..29fd0e03fb82ec 100644
--- a/clang/test/CodeGen/X86/x86_64-vector-abi-size.c
+++ b/clang/test/CodeGen/X86/x86_64-vector-abi-size.c
@@ -1,9 +1,12 @@
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,NOAVX,NOAVX512
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fclang-abi-compat=23 %s -o - | FileCheck %s --check-prefixes=CHECK,NOAVX,NOAVX512
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fexperimental-abi-lowering %s -o - | FileCheck %s --check-prefixes=CHECK,NOAVX,NOAVX512
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,AVX,NOAVX512
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx -emit-llvm -fexperimental-abi-lowering %s -o - | FileCheck %s --check-prefixes=CHECK,AVX,NOAVX512
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx512f -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
-// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx512f -emit-llvm -fexperimental-abi-lowering %s -o - | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,AVX,NOAVX512,DEFAULT
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx -emit-llvm -fclang-abi-compat=23 %s -o - | FileCheck %s --check-prefixes=CHECK,AVX,NOAVX512,DIRECT
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx -emit-llvm -fexperimental-abi-lowering %s -o - | FileCheck %s --check-prefixes=CHECK,AVX,NOAVX512,DIRECT
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx512f -emit-llvm %s -o - | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,DEFAULT
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx512f -emit-llvm -fclang-abi-compat=23 %s -o - | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,DIRECT
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +avx512f -emit-llvm -fexperimental-abi-lowering %s -o - | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,DIRECT
 
 typedef long double ld1 __attribute__((vector_size(16)));
 typedef long double ld2 __attribute__((vector_size(32)));
@@ -117,7 +120,11 @@ struct AB ab(struct AB s) { return s; }
 union __attribute__((aligned(32))) UA32 { ld1 v; };
 union UA32 ua32(union UA32 u) { return u; }
 // NOAVX: define{{.*}} void @ua32(ptr dead_on_unwind noalias writable sret(%union.UA32) align 32 %{{.*}}, ptr noundef byval(%union.UA32) align 32 %{{.*}})
-// AVX: define{{.*}} <4 x double> @ua32(<4 x double> %{{.*}})
+// The default classifier keeps the trailing eightbytes as NO_CLASS, making the
+// union MEMORY even with AVX. Clang 23 and experimental ABI lowering retain the
+// direct coercion when AVX is available.
+// DEFAULT: define{{.*}} void @ua32(ptr dead_on_unwind noalias writable sret(%union.UA32) align 32 %{{.*}}, ptr noundef byval(%union.UA32) align 32 %{{.*}})
+// DIRECT: define{{.*}} <4 x double> @ua32(<4 x double> %{{.*}})
 
 struct __attribute__((aligned(32))) SF32 { f3 v; };
 struct SF32 sf32(struct SF32 s) { return s; }



More information about the cfe-commits mailing list