[clang] [clang][Sema] Reject lax vector conversions involving `x87` long double (PR #228192)

Thu Oct 1 11:48:49 PDT 2026

llvmorg-github-actions[bot] wrote:




@llvm/pr-subscribers-clang

Author: Akash Manna (akash-manna-sky)

<details>
<summary>Changes</summary>

Fixes #173254
Fixes #63548

On x86, `long double` is an 80-bit x87 value stored in 16 bytes (12 on i386). `Sema::areVectorTypesSameSize` compares storage sizes, so it treated a one-element `long double` vector and a two-element `long long` vector as the same size, and the lax vector conversion rules inserted a `CK_BitCast` between them. LLVM gives `x86_fp80` a width of 80 bits, so CodeGen created an invalid `bitcast` and hit the `castIsValid` assertion. In #173254 the cast is on the argument of `__builtin_ia32_aesenc128`; in #63548 it is on the `return`, where the `__int128` vector produced by the comparison is converted back to the `long double` vector.

`areVectorTypesSameSize` now returns false when exactly one of the two element types is x87 `long double`, so these conversions are diagnosed as incompatible types instead of reaching CodeGen. Conversions where both sides are x87 `long double`, such as a one-element vector to the scalar, are still accepted, and `long double` in other formats is not affected.


---
Full diff: https://github.com/llvm/llvm-project/pull/228192.diff


4 Files Affected:

- (modified) clang/docs/ReleaseNotes.md (+1) 
- (modified) clang/lib/Sema/SemaExpr.cpp (+8) 
- (added) clang/test/Sema/vector-cast-x87.c (+46) 
- (modified) clang/test/SemaCXX/constexpr-builtin-bit-cast-fp80.cpp (+2-4) 


``````````diff

diff --git a/clang/docs/ReleaseNotes.md b/clang/docs/ReleaseNotes.md
index 3c6acf353f93f..7471abbc0fc51 100644
--- a/clang/docs/ReleaseNotes.md
+++ b/clang/docs/ReleaseNotes.md
@@ -533,6 +533,7 @@ features cannot lower the translation-unit ABI level;
 - Fixed incorrect handling of C++ import preprocessing token when a digraph character after import. (#GH190693)
 - Fixed a crash when emitting RTTI for a `dllexport` class, or the fundamental type descriptors for `__cxxabiv1::__fundamental_type_info`, under `-fvisibility=hidden`. (#GH207963)
 - Fixed an assertion failure when passing a wide string literal to `__builtin_nan`. (#GH212108)
+- Fixed an assertion failure when converting between an x87 `long double` vector and another vector type of the same size. (#GH173254), (#GH63548)
 - Fixed a constraint comparison bug in partial ordering. (#GH182671)
 - Fixed a rejected-valid case that used an explicit object parameter in an out-of-line definition of a nested class member. (#GH136472)
 - Fixed an assertion on omp taskloop transparent (#GH197162)
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index be1dc9f85d4f7..ceb827aa590f2 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -8070,6 +8070,14 @@ bool Sema::areVectorTypesSameSize(QualType SrcTy, QualType DestTy) {
   if (!breakDownVectorType(DestTy, DestLen, DestEltTy))
     return false;
 
+  // x87 long double has padding bits, so it cannot be bitcast to another type.
+  auto IsX87LongDouble = [&](QualType T) {
+    return T->isRealFloatingType() && &Context.getFloatTypeSemantics(T) ==
+                                          &llvm::APFloat::x87DoubleExtended();
+  };
+  if (IsX87LongDouble(SrcEltTy) != IsX87LongDouble(DestEltTy))
+    return false;
+
   // ASTContext::getTypeSize will return the size rounded up to a
   // power of 2, so instead of using that, we need to use the raw
   // element size multiplied by the element count.
diff --git a/clang/test/Sema/vector-cast-x87.c b/clang/test/Sema/vector-cast-x87.c
new file mode 100644
index 0000000000000..800a57ba9c245
--- /dev/null
+++ b/clang/test/Sema/vector-cast-x87.c
@@ -0,0 +1,46 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +aes -fsyntax-only -Wno-unused-value -verify %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -target-feature +aes -fsyntax-only -Wno-unused-value -mlong-double-128 -verify=quad %s
+// quad-no-diagnostics
+
+// GH173254
+typedef long double v16dl __attribute__ ((vector_size (16)));
+
+void foo(v16dl *a, v16dl *b) {
+    __builtin_ia32_aesenc128(*a, *b); // expected-error {{passing 'v16dl' (vector of 1 'long double' value) to parameter of incompatible type '__attribute__((__vector_size__(2 * sizeof(long long)))) long long' (vector of 2 'long long' values)}}
+}
+
+// GH63548
+typedef long double T __attribute__((vector_size(32)));
+
+T sum(T first, T second) { return first > second; } // expected-error {{returning '__attribute__((__vector_size__(2 * sizeof(__int128)))) __int128' (vector of 2 '__int128' values) from a function with incompatible result type 'T' (vector of 2 'long double' values)}}
+
+typedef double v16d __attribute__((vector_size(16)));
+typedef double v32d __attribute__((vector_size(32)));
+typedef long long v2ll __attribute__((vector_size(16)));
+typedef __int128 v1i128 __attribute__((vector_size(16)));
+
+v16d vector_cast(v16dl a) {
+  return (v16d)a; // expected-error {{invalid conversion between vector type 'v16d' (vector of 2 'double' values) and 'v16dl' (vector of 1 'long double' value) of different size}}
+}
+
+__int128 integer_cast(v16dl a) {
+  return (__int128)a; // expected-error {{invalid conversion between vector type 'v16dl' (vector of 1 'long double' value) and integer type '__int128' of different size}}
+}
+
+v2ll scalar_operand(v2ll a, long double b) {
+  return a + b; // expected-error {{cannot convert between scalar type 'long double' and vector type 'v2ll' (vector of 2 'long long' values) as implicit conversion would cause truncation}}
+}
+
+long double scalar_result(v1i128 a) {
+  return a; // expected-error {{returning 'v1i128' (vector of 1 '__int128' value) from a function with incompatible result type 'long double'}}
+}
+
+long double same_element(v16dl a) {
+  return a;
+}
+
+void foo_double(v16d *a, v16d *b) {
+    __builtin_ia32_aesenc128(*a, *b);
+}
+
+v32d sum_double(v32d first, v32d second) { return first > second; }
diff --git a/clang/test/SemaCXX/constexpr-builtin-bit-cast-fp80.cpp b/clang/test/SemaCXX/constexpr-builtin-bit-cast-fp80.cpp
index b37b362c81e75..840464668e028 100644
--- a/clang/test/SemaCXX/constexpr-builtin-bit-cast-fp80.cpp
+++ b/clang/test/SemaCXX/constexpr-builtin-bit-cast-fp80.cpp
@@ -56,11 +56,9 @@ namespace c_cast {
   // FIXME: re-enable the corresponding test cases in CodeGen/const-init.c when
   //  constexpr bitcast with x86_fp80 is supported
 
-  // expected-error at +2 {{constexpr variable 'b' must be initialized by a constant expression}}
-  // expected-note at +1 {{constexpr bit cast involving type 'long double' is not yet supported}}
+  // expected-error at +1 {{C-style cast from vector 'v2f80' (vector of 2 'long double' values) to vector 'v12i16' (vector of 12 'short' values) of different size}}
   constexpr static v12i16 b = (v12i16)(v2f80){1,2};
 
-  // expected-error at +2 {{constexpr variable 'c' must be initialized by a constant expression}}
-  // expected-note at +1 {{constexpr bit cast involving type 'long double' is not yet supported}}
+  // expected-error at +1 {{C-style cast from vector 'v12i16' (vector of 12 'short' values) to vector 'v2f80' (vector of 2 'long double' values) of different size}}
   constexpr static v2f80 c = (v2f80)(v12i16){0,0,0,-32768,16383,0,0,0,0,-32768,16384,0};
 }

``````````

</details>


https://github.com/llvm/llvm-project/pull/228192
