[clang] [llvm] [X86][Sema] Reject preserve_none calling convention on variadic functions (PR #220301)

[via llvm-commits]

https://github.com/AZero13 updated https://github.com/llvm/llvm-project/pull/220301

>From 7d603e68e29403df4370a0c8d4eeffa2f06799b7 Mon Sep 17 00:00:00 2001
From: AZero13 <gfunni234 at gmail.com>
Date: Tue, 1 Sep 2026 13:06:54 -0400
Subject: [PATCH] [X86][Sema] Reject preserve_none calling convention on
 variadic functions

The `preserve_none` calling convention is primarily optimized for JIT compilation,
hot paths, and avoiding caller-saved registers on non-variadic prototypes. On
x86-64, it delegates the 12th integer parameter to the `RAX` register.

For variadic functions on the x86-64 System V ABI, the caller is required to set
`%al` to the number of vector (XMM) registers being used to pass arguments.
Because `preserve_none` delegates unassigned parameters to the standard C calling
convention, `LowerCall` will emit an assignment of the XMM register count to
`%al`, clobbering the 12th argument passed in `%eax` (the lower 32 bits of
`RAX`).

Since `preserve_none` fundamentally lacks the register space to pass a 12th
argument in `RAX` while simultaneously passing the vector register count in `AL`,
this patch explicitly rejects `preserve_none` on variadic functions in both
Clang Sema and LLVM's X86 DAG lowering.
---
 clang/include/clang/Basic/Specifiers.h        |  1 +
 clang/test/Sema/preserve-none-call-conv.c     |  2 ++
 llvm/lib/Target/X86/X86ISelLoweringCall.cpp   | 14 ++++++++++++--
 llvm/test/CodeGen/X86/preserve_none_vararg.ll | 10 ++++++++++
 4 files changed, 25 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/X86/preserve_none_vararg.ll

diff --git a/clang/include/clang/Basic/Specifiers.h b/clang/include/clang/Basic/Specifiers.h
index c1b7198565f07..fdb13f62f1aa1 100644
--- a/clang/include/clang/Basic/Specifiers.h
+++ b/clang/include/clang/Basic/Specifiers.h
@@ -328,6 +328,7 @@ namespace clang {
     case CC_Swift:
     case CC_SwiftAsync:
     case CC_M68kRTD:
+    case CC_PreserveNone:
       return false;
     default:
       return true;
diff --git a/clang/test/Sema/preserve-none-call-conv.c b/clang/test/Sema/preserve-none-call-conv.c
index fc9463726e3f5..6c2f6245cdd57 100644
--- a/clang/test/Sema/preserve-none-call-conv.c
+++ b/clang/test/Sema/preserve-none-call-conv.c
@@ -18,3 +18,5 @@ typedef_fun_t typedef_fun_boo; // expected-note {{previous declaration is here}}
 void __attribute__((preserve_none)) typedef_fun_boo(int x) { } // expected-error {{function declared 'preserve_none' here was previously declared without calling convention}}
 
 struct type_test_boo {} __attribute__((preserve_none));  // expected-warning {{'preserve_none' attribute only applies to functions and function pointers}}
+
+void __attribute__((preserve_none)) variadic_boo(int x, ...) { } // expected-error {{variadic function cannot use 'preserve_none' calling convention}}
diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
index 9a03da14ee10e..a71cd95376046 100644
--- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
+++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
@@ -1960,7 +1960,11 @@ SDValue X86TargetLowering::LowerFormalArguments(
       MRI.disableCalleeSavedRegister(Pair.first);
   }
 
-  if (CallingConv::PreserveNone == CallConv)
+  if (CallingConv::PreserveNone == CallConv) {
+    if (isVarArg)
+      errorUnsupported(DAG, dl,
+                       "preserve_none calling convention does not support "
+                       "variadic functions");
     for (const ISD::InputArg &In : Ins) {
       if (In.Flags.isSwiftSelf() || In.Flags.isSwiftAsync() ||
           In.Flags.isSwiftError()) {
@@ -1969,6 +1973,7 @@ SDValue X86TargetLowering::LowerFormalArguments(
         break;
       }
     }
+  }
 
   return Chain;
 }
@@ -2777,7 +2782,11 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     InGlue = Chain.getValue(1);
   }
 
-  if (CallingConv::PreserveNone == CallConv)
+  if (CallingConv::PreserveNone == CallConv) {
+    if (isVarArg)
+      errorUnsupported(DAG, dl,
+                       "preserve_none calling convention does not support "
+                       "variadic functions");
     for (const ISD::OutputArg &Out : Outs) {
       if (Out.Flags.isSwiftSelf() || Out.Flags.isSwiftAsync() ||
           Out.Flags.isSwiftError()) {
@@ -2786,6 +2795,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
         break;
       }
     }
+  }
 
   // Handle result values, copying them out of physregs into vregs that we
   // return.
diff --git a/llvm/test/CodeGen/X86/preserve_none_vararg.ll b/llvm/test/CodeGen/X86/preserve_none_vararg.ll
new file mode 100644
index 0000000000000..508b65ef52946
--- /dev/null
+++ b/llvm/test/CodeGen/X86/preserve_none_vararg.ll
@@ -0,0 +1,10 @@
+; RUN: not llc -mtriple=x86_64-unknown-unknown < %s 2>&1 | FileCheck %s
+
+; CHECK: LLVM ERROR: preserve_none calling convention does not support variadic functions
+
+declare preserve_none void @vararg_func(i32, ...)
+
+define void @test() {
+  call preserve_none void (i32, ...) @vararg_func(i32 0)
+  ret void
+}