[llvm] [AArch64] Copy x4/x5 vararg payload into the x64 stack in Arm64EC exit thunks (PR #190933)
Alex Bradbury via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 8 01:54:02 PDT 2026
https://github.com/asb created https://github.com/llvm/llvm-project/pull/190933
Currently the x4/x5 in a variadic Arm64EC exit thunks are treated by LLVM like any other outgoing arguments. x4/x5 contain a pointer to the first stack parameter and the size of the parameters passed on the stack, and the generated exit thunk must memcpy these to the x86-64 stack. Current MSVC does this correctly.
Rather than introducing a new entry to the CallingConv enum, we mark the call as vararg in AArch64ArmECCallLowering so that the lowering logic in AArch64ISelLowering.cpp can recognise this case, perform the necessary memcpy, and drop the x4/x5 arguments.
LLVM should additionally ensure that x0-x3 are mirrored to f0-f3 in order to match the Windows x86-64 vararg ABI, but that change is left for a follow-up patch.
>From e09c9f449856abed3d76ad0745f3cdaee3e5c956 Mon Sep 17 00:00:00 2001
From: Alex Bradbury <asb at igalia.com>
Date: Wed, 8 Apr 2026 09:50:00 +0100
Subject: [PATCH] [AArch64] Copy x4/x5 vararg payload into the x64 stack in
Arm64EC exit thunks
Currently the x4/x5 in a variadic Arm64EC exit thunks are treated by LLVM
like any other outgoing arguments. x4/x5 contain a pointer to the first
stack parameter and the size of the parameters passed on the stack, and
the generated exit thunk must memcpy these to the x86-64 stack. Current
MSVC does this correctly.
Rather than introducing a new entry to the CallingConv enum, we mark the
call as vararg in AArch64ArmECCallLowering so that the lowering logic in
AArch64ISelLowering.cpp can recognise this case, perform the
necessary memcpy, and drop the x4/x5 arguments.
LLVM should additionally ensure that x0-x3 are mirrored to f0-f3 in
order to match the Windows x86-64 vararg ABI, but that change is left
for a follow-up patch.
---
.../AArch64/AArch64Arm64ECCallLowering.cpp | 10 +++-
.../Target/AArch64/AArch64ISelLowering.cpp | 59 ++++++++++++++++++-
.../CodeGen/AArch64/arm64ec-exit-thunks.ll | 42 +++++++++++--
.../AArch64/arm64ec-hybrid-patchable.ll | 20 ++++---
4 files changed, 112 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp b/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp
index 866e2f9c4218c..693b4dde9000a 100644
--- a/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp
@@ -436,6 +436,14 @@ Function *AArch64Arm64ECCallLowering::buildExitThunk(FunctionType *FT,
Value *Callee = IRB.CreateLoad(PtrTy, CalleePtr);
auto &DL = M->getDataLayout();
SmallVector<Value *> Args;
+ FunctionType *DispatcherCallTy = X64Ty;
+ // If we have a vararg function, the SelectionDAG lowering will need to
+ // recognize this so it can copy the arguments described by x4 (pointer) and
+ // x5 (length) to set up the x86-64 context correctly.
+ if (FT->isVarArg())
+ DispatcherCallTy =
+ FunctionType::get(X64Ty->getReturnType(), X64Ty->params(),
+ /*isVarArg=*/true);
// Pass the called function in x9.
auto X64TyOffset = 1;
@@ -487,7 +495,7 @@ Function *AArch64Arm64ECCallLowering::buildExitThunk(FunctionType *FT,
}
// FIXME: Transfer necessary attributes? sret? anything else?
- CallInst *Call = IRB.CreateCall(X64Ty, Callee, Args);
+ CallInst *Call = IRB.CreateCall(DispatcherCallTy, Callee, Args);
Call->setCallingConv(CallingConv::ARM64EC_Thunk_X64);
Value *RetVal = Call;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 165541ba089c8..2cb377a9e044c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -9616,6 +9616,18 @@ static void analyzeCallOperands(const AArch64TargetLowering &TLI,
CCInfo.AllocateStack(32, Align(16));
unsigned NumArgs = Outs.size();
+
+ // IsVarArg is only set on an ARM64EC_Thunk_X64 for exit thunks, so if set
+ // we know we have a vararg exit thunk where x4 and x5 must be consumed in
+ // this lowering (copying the data to the stack).
+ bool IsArm64ECVarArgExitThunk = CalleeCC == CallingConv::ARM64EC_Thunk_X64 &&
+ IsVarArg &&
+ !(CLI.CB && CLI.CB->isMustTailCall());
+ if (IsArm64ECVarArgExitThunk) {
+ assert(NumArgs >= 2 && "variadic Arm64EC exit thunk call missing x4/x5");
+ NumArgs -= 2;
+ }
+
for (unsigned i = 0; i != NumArgs; ++i) {
MVT ArgVT = Outs[i].VT;
ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
@@ -10264,6 +10276,38 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
AArch64ISD::SMSTOP, DL, DAG.getVTList(MVT::Other, MVT::Glue), Chain,
DAG.getTargetConstant((int32_t)(AArch64SVCR::SVCRZA), DL, MVT::i32));
+ auto PtrVT = getPointerTy(DAG.getDataLayout());
+ // If we have a variadic Arm64EC exit thunk, we must lower the x4/x5
+ // parameters (address and length of additional arguments) into an outgoing
+ // stack area.
+ bool IsArm64ECVarArgExitThunk = CallConv == CallingConv::ARM64EC_Thunk_X64 &&
+ IsVarArg &&
+ !(CLI.CB && CLI.CB->isMustTailCall());
+ SDValue ThunkVarArgSrc;
+ SDValue ThunkVarArgSize;
+ SDValue ThunkVarArgDst;
+ if (IsArm64ECVarArgExitThunk) {
+ // Materialize an aligned outgoing stack area now
+ // so the args described by x4 (pointer) and x5 (length) can be copied
+ // into a real x64-style stack layout.
+ assert(Outs.size() >= 2 && "variadic Arm64EC thunk call missing x4/x5");
+ assert(!IsTailCall &&
+ "tail calls are unsupported for variadic Arm64EC thunk calls");
+
+ ThunkVarArgSrc = OutVals[Outs.size() - 2];
+ ThunkVarArgSize = DAG.getZExtOrTrunc(OutVals[Outs.size() - 1], DL, PtrVT);
+ SDValue RoundedThunkVarArgSize = DAG.getNode(
+ ISD::ADD, DL, PtrVT, ThunkVarArgSize, DAG.getConstant(15, DL, PtrVT));
+ RoundedThunkVarArgSize =
+ DAG.getNode(ISD::AND, DL, PtrVT, RoundedThunkVarArgSize,
+ DAG.getSignedConstant(-16, DL, PtrVT));
+ ThunkVarArgDst = DAG.getNode(
+ ISD::DYNAMIC_STACKALLOC, DL, DAG.getVTList(PtrVT, MVT::Other),
+ {Chain, RoundedThunkVarArgSize, DAG.getConstant(0, DL, PtrVT)});
+ Chain = ThunkVarArgDst.getValue(1);
+ MFI.CreateVariableSizedObject(Align(16), nullptr);
+ }
+
// Adjust the stack pointer for the new arguments... and mark ZA uses.
// These operations are automatically eliminated by the prolog/epilog pass
assert((!IsSibCall || !ZAMarkerNode) && "ZA markers require CALLSEQ_START");
@@ -10286,7 +10330,6 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
SmallSet<unsigned, 8> RegsUsed;
SmallVector<SDValue, 8> MemOpChains;
- auto PtrVT = getPointerTy(DAG.getDataLayout());
if (IsVarArg && CLI.CB && CLI.CB->isMustTailCall()) {
const auto &Forwards = FuncInfo->getForwardedMustTailRegParms();
@@ -10298,7 +10341,8 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
// Walk the register/memloc assignments, inserting copies/loads.
unsigned ExtraArgLocs = 0;
- for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
+ unsigned NumThunkVarArgOperands = IsArm64ECVarArgExitThunk ? 2 : 0;
+ for (unsigned i = 0, e = Outs.size() - NumThunkVarArgOperands; i != e; ++i) {
CCValAssign &VA = ArgLocs[i - ExtraArgLocs];
SDValue Arg = OutVals[i];
ISD::ArgFlagsTy Flags = Outs[i].Flags;
@@ -10518,8 +10562,17 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
}
}
+ if (IsArm64ECVarArgExitThunk) {
+ SDValue Cpy = DAG.getMemcpy(
+ Chain, DL, ThunkVarArgDst, ThunkVarArgSrc, ThunkVarArgSize, Align(16),
+ /*isVol=*/false, /*AlwaysInline=*/false,
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo::getUnknownStack(MF),
+ MachinePointerInfo());
+ MemOpChains.push_back(Cpy);
+ }
+
if (IsVarArg && Subtarget->isWindowsArm64EC() &&
- !(CLI.CB && CLI.CB->isMustTailCall())) {
+ !(CLI.CB && CLI.CB->isMustTailCall()) && !IsArm64ECVarArgExitThunk) {
SDValue ParamPtr = StackPtr;
if (IsTailCall) {
// Create a dummy object at the top of the stack that can be used to get
diff --git a/llvm/test/CodeGen/AArch64/arm64ec-exit-thunks.ll b/llvm/test/CodeGen/AArch64/arm64ec-exit-thunks.ll
index 6fba6a3974574..d0a64b1bcf232 100644
--- a/llvm/test/CodeGen/AArch64/arm64ec-exit-thunks.ll
+++ b/llvm/test/CodeGen/AArch64/arm64ec-exit-thunks.ll
@@ -144,22 +144,52 @@ declare void @has_varargs(...) nounwind;
; CHECK-LABEL: .def $iexit_thunk$cdecl$v$varargs;
; CHECK: .section .wowthk$aa,"xr",discard,$iexit_thunk$cdecl$v$varargs
; CHECK: // %bb.0:
-; CHECK-NEXT: sub sp, sp, #64
-; CHECK-NEXT: .seh_stackalloc 64
+; CHECK-NEXT: stp x19, x20, [sp, #-64]! // 16-byte Folded Spill
+; CHECK-NEXT: .seh_save_regp_x x19, 64
+; CHECK-NEXT: stp x21, x22, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT: .seh_save_regp x21, 16
+; CHECK-NEXT: stp x25, x26, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT: .seh_save_regp x25, 32
; CHECK-NEXT: stp x29, x30, [sp, #48] // 16-byte Folded Spill
; CHECK-NEXT: .seh_save_fplr 48
; CHECK-NEXT: add x29, sp, #48
; CHECK-NEXT: .seh_add_fp 48
; CHECK-NEXT: .seh_endprologue
; CHECK-NEXT: adrp x8, __os_arm64x_dispatch_call_no_redirect
-; CHECK-NEXT: stp x4, x5, [sp, #32]
-; CHECK-NEXT: ldr x16, [x8, :lo12:__os_arm64x_dispatch_call_no_redirect]
+; CHECK-NEXT: mov x19, x3
+; CHECK-NEXT: mov x20, x2
+; CHECK-NEXT: ldr x25, [x8, :lo12:__os_arm64x_dispatch_call_no_redirect]
+; CHECK-NEXT: add x8, x5, #15
+; CHECK-NEXT: mov x21, x1
+; CHECK-NEXT: lsr x15, x8, #4
+; CHECK-NEXT: mov x22, x0
+; CHECK-NEXT: mov x26, x9
+; CHECK-NEXT: bl "#__chkstk_arm64ec"
+; CHECK-NEXT: sub x0, sp, x15, lsl #4
+; CHECK-NEXT: mov sp, x0
+; CHECK-NEXT: sub sp, sp, #32
+; CHECK-NEXT: mov x1, x4
+; CHECK-NEXT: mov x2, x5
+; CHECK-NEXT: bl "#memcpy"
+; CHECK-NEXT: mov x9, x26
+; CHECK-NEXT: mov x0, x22
+; CHECK-NEXT: mov x1, x21
+; CHECK-NEXT: mov x2, x20
+; CHECK-NEXT: mov x3, x19
+; CHECK-NEXT: mov x16, x25
; CHECK-NEXT: blr x16
+; CHECK-NEXT: add sp, sp, #32
; CHECK-NEXT: .seh_startepilogue
+; CHECK-NEXT: sub sp, x29, #48
+; CHECK-NEXT: .seh_add_fp 48
; CHECK-NEXT: ldp x29, x30, [sp, #48] // 16-byte Folded Reload
; CHECK-NEXT: .seh_save_fplr 48
-; CHECK-NEXT: add sp, sp, #64
-; CHECK-NEXT: .seh_stackalloc 64
+; CHECK-NEXT: ldp x25, x26, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT: .seh_save_regp x25, 32
+; CHECK-NEXT: ldp x21, x22, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT: .seh_save_regp x21, 16
+; CHECK-NEXT: ldp x19, x20, [sp], #64 // 16-byte Folded Reload
+; CHECK-NEXT: .seh_save_regp_x x19, 64
; CHECK-NEXT: .seh_endepilogue
; CHECK-NEXT: ret
; CHECK-NEXT: .seh_endfunclet
diff --git a/llvm/test/CodeGen/AArch64/arm64ec-hybrid-patchable.ll b/llvm/test/CodeGen/AArch64/arm64ec-hybrid-patchable.ll
index 7c77832a9d9a5..b6599e63e69de 100644
--- a/llvm/test/CodeGen/AArch64/arm64ec-hybrid-patchable.ll
+++ b/llvm/test/CodeGen/AArch64/arm64ec-hybrid-patchable.ll
@@ -321,15 +321,17 @@ define dso_local void @caller() nounwind {
; SYM: [122](sec 0)(fl 0x00)(ty 0)(scl 69) (nx 1) 0x00000000 has_varargs
; SYM-NEXT: AUX indx 124 srch 3
; SYM-NEXT: [124](sec 0)(fl 0x00)(ty 20)(scl 2) (nx 0) 0x00000000 EXP+#has_varargs
-; SYM-NEXT: [125](sec 0)(fl 0x00)(ty 0)(scl 69) (nx 1) 0x00000000 has_sret
-; SYM-NEXT: AUX indx 127 srch 3
-; SYM-NEXT: [127](sec 0)(fl 0x00)(ty 20)(scl 2) (nx 0) 0x00000000 EXP+#has_sret
-; SYM-NEXT: [128](sec 0)(fl 0x00)(ty 0)(scl 69) (nx 1) 0x00000000 exp
-; SYM-NEXT: AUX indx 130 srch 3
-; SYM-NEXT: [130](sec 0)(fl 0x00)(ty 20)(scl 2) (nx 0) 0x00000000 EXP+#exp
-; SYM-NEXT: [131](sec 0)(fl 0x00)(ty 0)(scl 69) (nx 1) 0x00000000 #has_varargs
+; SYM-NEXT: [125](sec 0)(fl 0x00)(ty 0)(scl 2) (nx 0) 0x00000000 #__chkstk_arm64ec
+; SYM-NEXT: [126](sec 0)(fl 0x00)(ty 0)(scl 2) (nx 0) 0x00000000 #memcpy
+; SYM-NEXT: [127](sec 0)(fl 0x00)(ty 0)(scl 69) (nx 1) 0x00000000 has_sret
+; SYM-NEXT: AUX indx 129 srch 3
+; SYM-NEXT: [129](sec 0)(fl 0x00)(ty 20)(scl 2) (nx 0) 0x00000000 EXP+#has_sret
+; SYM-NEXT: [130](sec 0)(fl 0x00)(ty 0)(scl 69) (nx 1) 0x00000000 exp
+; SYM-NEXT: AUX indx 132 srch 3
+; SYM-NEXT: [132](sec 0)(fl 0x00)(ty 20)(scl 2) (nx 0) 0x00000000 EXP+#exp
+; SYM-NEXT: [133](sec 0)(fl 0x00)(ty 0)(scl 69) (nx 1) 0x00000000 #has_varargs
; SYM-NEXT: AUX indx 58 srch 3
-; SYM-NEXT: [133](sec 0)(fl 0x00)(ty 0)(scl 69) (nx 1) 0x00000000 #has_sret
+; SYM-NEXT: [135](sec 0)(fl 0x00)(ty 0)(scl 69) (nx 1) 0x00000000 #has_sret
; SYM-NEXT: AUX indx 68 srch 3
-; SYM-NEXT: [135](sec 0)(fl 0x00)(ty 0)(scl 69) (nx 1) 0x00000000 #exp
+; SYM-NEXT: [137](sec 0)(fl 0x00)(ty 0)(scl 69) (nx 1) 0x00000000 #exp
; SYM-NEXT: AUX indx 78 srch 3
More information about the llvm-commits
mailing list