[llvm] [X86] Fix invalid codegen when passing f80 arguments in __vectorcall (PR #203913)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 29 18:07:56 PDT 2026
https://github.com/AZero13 updated https://github.com/llvm/llvm-project/pull/203913
>From 15c26c019e8ebeeb4c33524d0fa26950b2292f6e Mon Sep 17 00:00:00 2001
From: AZero13 <gfunni234 at gmail.com>
Date: Wed, 29 Jul 2026 20:40:42 -0400
Subject: [PATCH] [X86] Fix vectorcalls for fp80, fp16, and bloat16
The `__vectorcall` calling convention allocates SSE registers (`XMM0`-`XMM5`) for floating-point types and vector types. However, LLVM's vectorcall assignment logic was previously checking if an argument was a floating-point type simply by calling `ValVT.isFloatingPoint()`. Since `f80` is classified as a floating-point type, LLVM incorrectly assigned an `XMM` register for the argument. Later during compilation, the backend would crash because there is no valid hardware instruction to copy an `XMM` register directly to an x87 stack register (`FP0`-`FP6`).
---
llvm/lib/Target/X86/X86CallingConv.cpp | 21 ++++++++++----
llvm/lib/Target/X86/X86CallingConv.td | 4 +--
.../CodeGen/X86/vectorcall-f80-tailcall.ll | 29 +++++++++++++++++++
llvm/test/CodeGen/X86/vectorcall-f80.ll | 22 ++++++++++++++
.../CodeGen/X86/vectorcall-other-floats.ll | 21 ++++++++++++++
5 files changed, 89 insertions(+), 8 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/vectorcall-f80-tailcall.ll
create mode 100644 llvm/test/CodeGen/X86/vectorcall-f80.ll
create mode 100644 llvm/test/CodeGen/X86/vectorcall-other-floats.ll
diff --git a/llvm/lib/Target/X86/X86CallingConv.cpp b/llvm/lib/Target/X86/X86CallingConv.cpp
index ce2ce1cbc6379..7ecfe699b93f9 100644
--- a/llvm/lib/Target/X86/X86CallingConv.cpp
+++ b/llvm/lib/Target/X86/X86CallingConv.cpp
@@ -87,6 +87,18 @@ static ArrayRef<MCPhysReg> CC_X86_64_VectorCallGetGPRs() {
return RegListGPR;
}
+/// Return true if \p VT is a vectorcall vector type.
+///
+/// Per the vectorcall spec, vector types are float, double, or SIMD types such
+/// as __m128/__m256. x86_fp80 is an x87 type and must use the non-vector rules
+/// (see CC_X86_Win64_C). Argument handling matches RetCC_X86_*_VectorCall in
+/// X86CallingConv.td, which also passes f128 in XMM registers.
+static bool CC_X86_IsVectorCallVectorType(MVT VT) {
+ return VT == MVT::f16 || VT == MVT::bf16 || VT == MVT::f32 ||
+ VT == MVT::f64 || VT == MVT::f128 ||
+ (VT.isVector() && VT.getSizeInBits() >= 128);
+}
+
static bool CC_X86_VectorCallAssignRegister(unsigned &ValNo, MVT &ValVT,
MVT &LocVT,
CCValAssign::LocInfo &LocInfo,
@@ -139,8 +151,7 @@ static bool CC_X86_64_VectorCall(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
// Process only vector types as defined by vectorcall spec:
// "A vector type is either a floating-point type, for example,
// a float or double, or an SIMD vector type, for example, __m128 or __m256".
- if (!(ValVT.isFloatingPoint() ||
- (ValVT.isVector() && ValVT.getSizeInBits() >= 128))) {
+ if (!CC_X86_IsVectorCallVectorType(ValVT)) {
// If R9 was already assigned it means that we are after the fourth element
// and because this is not an HVA / Vector type, we need to allocate
// shadow XMM register.
@@ -199,10 +210,8 @@ static bool CC_X86_32_VectorCall(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
// Process only vector types as defined by vectorcall spec:
// "A vector type is either a floating point type, for example,
// a float or double, or an SIMD vector type, for example, __m128 or __m256".
- if (!(ValVT.isFloatingPoint() ||
- (ValVT.isVector() && ValVT.getSizeInBits() >= 128))) {
+ if (!CC_X86_IsVectorCallVectorType(ValVT))
return false;
- }
if (ArgFlags.isHva())
return true; // If this is an HVA - Stop the search.
@@ -216,7 +225,7 @@ static bool CC_X86_32_VectorCall(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
// In case we did not find an available XMM register for a vector -
// pass it indirectly.
// It is similar to CCPassIndirect, with the addition of inreg.
- if (!ValVT.isFloatingPoint()) {
+ if (ValVT.isVector()) {
LocVT = MVT::i32;
LocInfo = CCValAssign::Indirect;
ArgFlags.setInReg();
diff --git a/llvm/lib/Target/X86/X86CallingConv.td b/llvm/lib/Target/X86/X86CallingConv.td
index bde5f81eaa7b8..ff8acbb7f4ae3 100644
--- a/llvm/lib/Target/X86/X86CallingConv.td
+++ b/llvm/lib/Target/X86/X86CallingConv.td
@@ -355,7 +355,7 @@ def RetCC_X86_32_HiPE : CallingConv<[
// X86-32 Vectorcall return-value convention.
def RetCC_X86_32_VectorCall : CallingConv<[
// Floating Point types are returned in XMM0,XMM1,XMMM2 and XMM3.
- CCIfType<[f32, f64, f128],
+ CCIfType<[f16, bf16, f32, f64, f128],
CCAssignToReg<[XMM0,XMM1,XMM2,XMM3]>>,
// Return integers in the standard way.
@@ -392,7 +392,7 @@ def RetCC_X86_Win64_C : CallingConv<[
// X86-64 vectorcall return-value convention.
def RetCC_X86_64_Vectorcall : CallingConv<[
// Vectorcall calling convention always returns FP values in XMMs.
- CCIfType<[f32, f64, f128],
+ CCIfType<[f16, bf16, f32, f64, f128],
CCAssignToReg<[XMM0, XMM1, XMM2, XMM3]>>,
// Otherwise, everything is the same as Windows X86-64 C CC.
diff --git a/llvm/test/CodeGen/X86/vectorcall-f80-tailcall.ll b/llvm/test/CodeGen/X86/vectorcall-f80-tailcall.ll
new file mode 100644
index 0000000000000..984290821a50a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vectorcall-f80-tailcall.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -O2 -mtriple=i686-pc-windows-gnu < %s | FileCheck %s
+
+; Verify that tail calls to vectorcall functions with two x86_fp80 arguments
+; pass arguments in the correct stack order (X at lower address, Y above).
+
+declare x86_vectorcallcc x86_fp80 @tailcall_f80_callee(x86_fp80, x86_fp80)
+
+define dso_local x86_vectorcallcc x86_fp80 @tailcall_f80_swap_args(x86_fp80 %Y, x86_fp80 %X) {
+; CHECK-LABEL: tailcall_f80_swap_args@@24:
+; CHECK: # %bb.0:
+; CHECK-DAG: fldt 40(%esp)
+; CHECK-DAG: fldt 28(%esp)
+; CHECK-DAG: fstpt (%esp)
+; CHECK-DAG: fstpt 12(%esp)
+; CHECK-NOT: xmm
+; CHECK: calll tailcall_f80_callee@@24
+entry:
+ %ret = tail call x86_vectorcallcc x86_fp80 @tailcall_f80_callee(x86_fp80 %X, x86_fp80 %Y)
+ ret x86_fp80 %ret
+}
+
+define dso_local x86_vectorcallcc x86_fp80 @tailcall_f80_passthrough(x86_fp80 %X, x86_fp80 %Y) {
+; CHECK-LABEL: tailcall_f80_passthrough@@24:
+; CHECK: jmp tailcall_f80_callee@@24
+entry:
+ %ret = tail call x86_vectorcallcc x86_fp80 @tailcall_f80_callee(x86_fp80 %X, x86_fp80 %Y)
+ ret x86_fp80 %ret
+}
diff --git a/llvm/test/CodeGen/X86/vectorcall-f80.ll b/llvm/test/CodeGen/X86/vectorcall-f80.ll
new file mode 100644
index 0000000000000..3a50d47524de0
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vectorcall-f80.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=i686-pc-windows-gnu < %s | FileCheck %s
+; RUN: llc -mtriple=x86_64-linux-gnu < %s | FileCheck %s --check-prefix=LINUX64
+
+; Verify that x86_fp80 arguments and return values are handled correctly
+; and do not crash the backend when using the vectorcall calling convention.
+
+define dso_local x86_vectorcallcc x86_fp80 @vectorcall_f80(x86_fp80 %A) {
+; CHECK-LABEL: vectorcall_f80@@12:
+; CHECK: # %bb.0:
+; CHECK-NEXT: fldt 4(%esp)
+; CHECK-NEXT: retl $12
+;
+; LINUX64-LABEL: "vectorcall_f80@@16":
+; LINUX64: .cfi_startproc
+; LINUX64-NEXT: # %bb.0:
+; LINUX64-NEXT: fldt
+; LINUX64-SAME: (%rcx)
+; LINUX64-NEXT: retq
+entry:
+ ret x86_fp80 %A
+}
diff --git a/llvm/test/CodeGen/X86/vectorcall-other-floats.ll b/llvm/test/CodeGen/X86/vectorcall-other-floats.ll
new file mode 100644
index 0000000000000..a133c7eebf0ac
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vectorcall-other-floats.ll
@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=i686-pc-windows-gnu -mattr=+avx512f,+avx512vl,+avx512bw,+avx512fp16 < %s | FileCheck %s
+; RUN: llc -mtriple=x86_64-pc-windows-msvc -mattr=+avx512f,+avx512vl,+avx512bw,+avx512fp16 < %s | FileCheck %s --check-prefix=X64
+
+; Verify that half (f16), bfloat (bf16), and fp128 arguments and return values
+; are assigned to XMM registers when using the vectorcall calling convention.
+
+define dso_local x86_vectorcallcc half @vectorcall_f16(half %A) {
+entry:
+ ret half %A
+}
+
+define dso_local x86_vectorcallcc bfloat @vectorcall_bf16(bfloat %A) {
+entry:
+ ret bfloat %A
+}
+
+define dso_local x86_vectorcallcc fp128 @vectorcall_f128(fp128 %A) {
+entry:
+ ret fp128 %A
+}
More information about the llvm-commits
mailing list