[llvm] [AArch64] support `byval` arguments in `tailcc` tail calls (PR #206718)
Folkert de Vries via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 30 05:25:51 PDT 2026
https://github.com/folkertdev updated https://github.com/llvm/llvm-project/pull/206718
>From 47a374ea43fb3f4d249e5ee1556ae26b997fa615 Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Sun, 28 Jun 2026 20:23:21 +0200
Subject: [PATCH 01/11] copy `ByValCopyKind` from the X86 backend
---
.../Target/AArch64/AArch64ISelLowering.cpp | 53 +++++++++++++++++--
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 17 ++++++
2 files changed, 65 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 67ef911117eff..0a2841e482f8b 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -9690,14 +9690,12 @@ bool AArch64TargetLowering::isEligibleForTailCallOptimization(
return false;
// Byval parameters hand the function a pointer directly into the stack area
- // we want to reuse during a tail call. Working around this *is* possible (see
- // X86) but less efficient and uglier in LowerCall.
+ // we want to reuse during a tail call. We route such arguments via a
+ // temporary in the current frame, so a function with byval arguments can
+ // still be tail-called.
for (Function::const_arg_iterator i = CallerF.arg_begin(),
e = CallerF.arg_end();
i != e; ++i) {
- if (i->hasByValAttr())
- return false;
-
// On Windows, "inreg" attributes signify non-aggregate indirect returns.
// In this case, it is necessary to save X0/X1 in the callee and return it
// in X0. Tail call opt may interfere with this, so we disable tail call
@@ -9979,6 +9977,51 @@ getSMToggleCondition(const SMECallAttrs &CallAttrs) {
llvm_unreachable("Unsupported attributes");
}
+// Returns the type of copying which is required to set up a byval argument to
+// a tail-called function. This isn't needed for non-tail calls, because they
+// always need the equivalent of CopyOnce, but tail-calls sometimes need two to
+// avoid clobbering another argument (CopyViaTemp), and sometimes can be
+// optimised to zero copies when forwarding an argument from the caller's
+// caller (NoCopy).
+AArch64TargetLowering::ByValCopyKind
+AArch64TargetLowering::ByValNeedsCopyForTailCall(SelectionDAG &DAG, SDValue Src,
+ SDValue Dst,
+ ISD::ArgFlagsTy Flags) const {
+ MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
+
+ // Globals are always safe to copy from.
+ if (isa<GlobalAddressSDNode>(Src) || isa<ExternalSymbolSDNode>(Src))
+ return CopyOnce;
+
+ // Can only analyse frame index nodes, conservatively assume we need a
+ // temporary.
+ auto *SrcFrameIdxNode = dyn_cast<FrameIndexSDNode>(Src);
+ auto *DstFrameIdxNode = dyn_cast<FrameIndexSDNode>(Dst);
+ if (!SrcFrameIdxNode || !DstFrameIdxNode)
+ return CopyViaTemp;
+
+ int SrcFI = SrcFrameIdxNode->getIndex();
+ int DstFI = DstFrameIdxNode->getIndex();
+ assert(MFI.isFixedObjectIndex(DstFI) &&
+ "byval passed in non-fixed stack slot");
+
+ int64_t SrcOffset = MFI.getObjectOffset(SrcFI);
+ int64_t DstOffset = MFI.getObjectOffset(DstFI);
+
+ // If the source is in the local frame, then the copy to the argument memory
+ // is always valid.
+ bool FixedSrc = MFI.isFixedObjectIndex(SrcFI);
+ if (!FixedSrc || (FixedSrc && SrcOffset < 0))
+ return CopyOnce;
+
+ // If the value is already in the correct location, then no copying is
+ // needed. If not, then we need to copy via a temporary.
+ if (SrcOffset == DstOffset)
+ return NoCopy;
+ else
+ return CopyViaTemp;
+}
+
/// Check whether a stack argument requires lowering in a tail call.
static bool shouldLowerTailCallStackArg(const MachineFunction &MF,
const CCValAssign &VA, SDValue Arg,
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 704eed7877bdc..f5facc64c4020 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -62,6 +62,19 @@ enum : unsigned { PTR32_SPTR = 270, PTR32_UPTR = 271, PTR64 = 272 };
class AArch64Subtarget;
class AArch64TargetLowering : public TargetLowering {
+ // Copying needed for an outgoing byval argument.
+ enum ByValCopyKind {
+ // Argument is already in the correct location, no copy needed.
+ NoCopy,
+ // Argument value is currently in the local stack frame, needs copying to
+ // outgoing arguemnt area.
+ CopyOnce,
+ // Argument value is currently in the outgoing argument area, but not at
+ // the correct offset, so needs copying via a temporary in local stack
+ // space.
+ CopyViaTemp,
+ };
+
public:
explicit AArch64TargetLowering(const TargetMachine &TM,
const AArch64Subtarget &STI);
@@ -601,6 +614,10 @@ class AArch64TargetLowering : public TargetLowering {
// register-name matcher, shared with getRegisterByName.
Register matchRegisterName(StringRef RegName) const;
+ ByValCopyKind ByValNeedsCopyForTailCall(SelectionDAG &DAG, SDValue Src,
+ SDValue Dst,
+ ISD::ArgFlagsTy Flags) const;
+
private:
/// Keep a pointer to the AArch64Subtarget around so that we can
/// make the right decision when generating code for different targets.
>From 9f45159bf617a6c299b330fb8d2d8aed47def8bd Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Sun, 28 Jun 2026 20:37:35 +0200
Subject: [PATCH 02/11] copy lowerCall logic
---
.../Target/AArch64/AArch64ISelLowering.cpp | 86 +++++++++++++++++++
1 file changed, 86 insertions(+)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 0a2841e482f8b..3f143e992a715 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -10061,6 +10061,20 @@ static bool shouldLowerTailCallStackArg(const MachineFunction &MF,
return true;
}
+/// Make a copy of an aggregate at address specified by "Src" to address
+/// "Dst" with size and alignment information specified by the specific
+/// parameter attribute. The copy will be passed as a byval function parameter.
+static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
+ SDValue Chain, ISD::ArgFlagsTy Flags,
+ SelectionDAG &DAG, const SDLoc &dl) {
+ SDValue SizeNode = DAG.getIntPtrConstant(Flags.getByValSize(), dl);
+ Align Alignment = Flags.getNonZeroByValAlign();
+ return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Alignment, Alignment,
+ /*isVolatile*/ false, /*AlwaysInline=*/true,
+ /*CI=*/nullptr, std::nullopt, MachinePointerInfo(),
+ MachinePointerInfo());
+}
+
/// LowerCall - Lower a call to a callseq_start + CALL + callseq_end chain,
/// and add input and output parameter nodes.
SDValue
@@ -10203,6 +10217,74 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
assert(FPDiff % 16 == 0 && "unaligned stack on tail call");
}
+ // If we are doing a tail-call, any byval arguments will be written to stack
+ // space which was used for incoming arguments. If any the values being used
+ // are incoming byval arguments to this function, then they might be
+ // overwritten by the stores of the outgoing arguments. To avoid this, we
+ // need to make a temporary copy of them in local stack space, then copy back
+ // to the argument area.
+ // FIXME: There's potential to improve the code by using virtual registers for
+ // temporary storage, and letting the register allocator spill if needed.
+ SmallVector<SDValue, 8> ByValTemporaries;
+ SDValue ByValTempChain;
+ if (IsTailCall) {
+ // Use null SDValue to mean "no temporary recorded for this arg index".
+ ByValTemporaries.assign(OutVals.size(), SDValue());
+
+ SmallVector<SDValue, 8> ByValCopyChains;
+ for (const CCValAssign &VA : ArgLocs) {
+ unsigned ArgIdx = VA.getValNo();
+ SDValue Src = OutVals[ArgIdx];
+ ISD::ArgFlagsTy Flags = Outs[ArgIdx].Flags;
+
+ if (!Flags.isByVal())
+ continue;
+
+ auto PtrVT = getPointerTy(DAG.getDataLayout());
+
+ // Destination: where this byval should live in the callee’s frame
+ // after the tail call.
+ int64_t Offset = VA.getLocMemOffset() + FPDiff;
+ uint64_t Size = VA.getLocVT().getFixedSizeInBits() / 8;
+ int FI = MF.getFrameInfo().CreateFixedObject(Size, Offset,
+ /*IsImmutable=*/true);
+ SDValue Dst = DAG.getFrameIndex(FI, PtrVT);
+
+ ByValCopyKind Copy = ByValNeedsCopyForTailCall(DAG, Src, Dst, Flags);
+
+ if (Copy == NoCopy) {
+ // If the argument is already at the correct offset on the stack
+ // (because we are forwarding a byval argument from our caller), we
+ // don't need any copying.
+ continue;
+ } else if (Copy == CopyOnce) {
+ // If the argument is in our local stack frame, no other argument
+ // preparation can clobber it, so we can copy it to the final location
+ // later.
+ ByValTemporaries[ArgIdx] = Src;
+ } else {
+ assert(Copy == CopyViaTemp && "unexpected enum value");
+ // If we might be copying this argument from the outgoing argument
+ // stack area, we need to copy via a temporary in the local stack
+ // frame.
+ MachineFrameInfo &MFI = MF.getFrameInfo();
+ int TempFrameIdx = MFI.CreateStackObject(Flags.getByValSize(),
+ Flags.getNonZeroByValAlign(),
+ /*isSS=*/false);
+ SDValue Temp =
+ DAG.getFrameIndex(TempFrameIdx, getPointerTy(DAG.getDataLayout()));
+
+ SDValue CopyChain =
+ CreateCopyOfByValArgument(Src, Temp, Chain, Flags, DAG, DL);
+ ByValCopyChains.push_back(CopyChain);
+ ByValTemporaries[ArgIdx] = Temp;
+ }
+ }
+ if (!ByValCopyChains.empty())
+ ByValTempChain =
+ DAG.getNode(ISD::TokenFactor, DL, MVT::Other, ByValCopyChains);
+ }
+
auto DescribeCallsite =
[&](OptimizationRemarkAnalysis &R) -> OptimizationRemarkAnalysis & {
R << "call from '" << ore::NV("Caller", MF.getName()) << "' to '";
@@ -10261,6 +10343,10 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
}
}
+ if (ByValTempChain)
+ Chain =
+ DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chain, ByValTempChain);
+
// Walk the register/memloc assignments, inserting copies/loads.
unsigned ExtraArgLocs = 0;
for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
>From caf893bcbf766dbf6097180b896c77b7d4b9dd4b Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Sun, 28 Jun 2026 21:11:22 +0200
Subject: [PATCH 03/11] use the byval temporary
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 17 +++++++++++++----
1 file changed, 13 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 3f143e992a715..4e2c78c94d62e 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -10523,10 +10523,16 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
int32_t Offset = LocMemOffset + BEAlign;
if (IsTailCall) {
- // When the frame pointer is perfectly aligned for the tail call and the
- // same stack argument is passed down intact, we can reuse it.
- if (!FPDiff && !shouldLowerTailCallStackArg(MF, VA, Arg, Flags, Offset))
+ if (Flags.isByVal()) {
+ if (!ByValTemporaries[i])
+ // The argument is already in the right place.
+ continue;
+ } else if (!FPDiff &&
+ !shouldLowerTailCallStackArg(MF, VA, Arg, Flags, Offset)) {
+ // When the frame pointer is perfectly aligned for the tail call and
+ // the same stack argument is passed down intact, we can reuse it.
continue;
+ }
Offset = Offset + FPDiff;
int FI = MF.getFrameInfo().CreateFixedObject(OpSize, Offset, true);
@@ -10546,10 +10552,13 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
}
if (Outs[i].Flags.isByVal()) {
+ // For tail calls, copy from the (possibly staged) source chosen by the
+ // pre-pass above; the temporary never overlaps the destination.
+ SDValue ByValSrc = IsTailCall ? ByValTemporaries[i] : Arg;
SDValue SizeNode =
DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i64);
SDValue Cpy = DAG.getMemcpy(
- Chain, DL, DstAddr, Arg, SizeNode,
+ Chain, DL, DstAddr, ByValSrc, SizeNode,
Outs[i].Flags.getNonZeroByValAlign(),
Outs[i].Flags.getNonZeroByValAlign(),
/*isVol = */ false, /*AlwaysInline = */ false,
>From 1ad39e55a82200625ceb2b7f2dd0b46bfe6644f6 Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Sun, 28 Jun 2026 23:10:40 +0200
Subject: [PATCH 04/11] update tests for
`llvm/test/CodeGen/AArch64/tail-call.ll`
---
llvm/test/CodeGen/AArch64/tail-call.ll | 143 ++++++++++++++++---------
1 file changed, 95 insertions(+), 48 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/tail-call.ll b/llvm/test/CodeGen/AArch64/tail-call.ll
index 1a5599fcc508b..c935a90dddf2d 100644
--- a/llvm/test/CodeGen/AArch64/tail-call.ll
+++ b/llvm/test/CodeGen/AArch64/tail-call.ll
@@ -1,5 +1,6 @@
-; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -tailcallopt | FileCheck %s --check-prefixes=SDAG,COMMON
-; RUN: llc -global-isel -global-isel-abort=1 -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -tailcallopt | FileCheck %s --check-prefixes=GISEL,COMMON
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -tailcallopt | FileCheck %s --check-prefixes=COMMON,SDAG
+; RUN: llc -global-isel -global-isel-abort=1 -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -tailcallopt | FileCheck %s --check-prefixes=COMMON,GISEL
declare fastcc void @callee_stack0()
declare fastcc void @callee_stack8([8 x i64], i64)
@@ -8,109 +9,137 @@ declare extern_weak fastcc void @callee_weak()
define fastcc void @caller_to0_from0() nounwind {
; COMMON-LABEL: caller_to0_from0:
-; COMMON-NEXT: // %bb.
+; COMMON: // %bb.0:
+; COMMON-NEXT: b callee_stack0
tail call fastcc void @callee_stack0()
ret void
-; COMMON-NEXT: b callee_stack0
}
define fastcc void @caller_to0_from8([8 x i64], i64) #0 {
; COMMON-LABEL: caller_to0_from8:
+; COMMON: // %bb.0:
+; COMMON-NEXT: add sp, sp, #16
+; COMMON-NEXT: .cfi_def_cfa_offset -16
+; COMMON-NEXT: b callee_stack0
tail call fastcc void @callee_stack0()
ret void
-; COMMON: add sp, sp, #16
-; COMMON: .cfi_def_cfa_offset -16
-; COMMON-NEXT: b callee_stack0
}
define fastcc void @caller_to8_from0() #0 {
; COMMON-LABEL: caller_to8_from0:
+; COMMON: // %bb.0:
+; COMMON-NEXT: mov w8, #42 // =0x2a
+; COMMON-NEXT: str x8, [sp, #-16]!
+; COMMON-NEXT: .cfi_def_cfa_offset 16
+; COMMON-NEXT: b callee_stack8
; Key point is that the "42" should go #16 below incoming stack
; pointer (we didn't have arg space to reuse).
tail call fastcc void @callee_stack8([8 x i64] undef, i64 42)
ret void
-; COMMON: str {{x[0-9]+}}, [sp, #-16]!
-; COMMON-NEXT: .cfi_def_cfa_offset 16
-; COMMON-NEXT: b callee_stack8
}
define fastcc void @caller_to8_from8([8 x i64], i64 %a) #0 {
; COMMON-LABEL: caller_to8_from8:
-; COMMON-NOT: sub sp,
+; COMMON: // %bb.0:
+; COMMON-NEXT: mov w8, #42 // =0x2a
+; COMMON-NEXT: str x8, [sp]
+; COMMON-NEXT: b callee_stack8
; Key point is that the "%a" should go where at SP on entry.
tail call fastcc void @callee_stack8([8 x i64] undef, i64 42)
ret void
-; COMMON: str {{x[0-9]+}}, [sp]
-; COMMON-NEXT: b callee_stack8
}
define fastcc void @caller_to16_from8([8 x i64], i64 %a) #0 {
-; COMMON-LABEL: caller_to16_from8:
-; COMMON-NOT: sub sp,
-
; Important point is that the call reuses the "dead" argument space
; above %a on the stack. If it tries to go below incoming-SP then the
; callee will not deallocate the space, even in fastcc.
+; SDAG-LABEL: caller_to16_from8:
+; SDAG: // %bb.0:
+; SDAG-NEXT: mov w8, #2 // =0x2
+; SDAG-NEXT: mov w9, #42 // =0x2a
+; SDAG-NEXT: stp x9, x8, [sp]
+; SDAG-NEXT: b callee_stack16
+;
+; GISEL-LABEL: caller_to16_from8:
+; GISEL: // %bb.0:
+; GISEL-NEXT: mov w8, #42 // =0x2a
+; GISEL-NEXT: mov w9, #2 // =0x2
+; GISEL-NEXT: stp x8, x9, [sp]
+; GISEL-NEXT: b callee_stack16
tail call fastcc void @callee_stack16([8 x i64] undef, i64 42, i64 2)
-; COMMON: stp {{x[0-9]+}}, {{x[0-9]+}}, [sp]
-; COMMON-NEXT: b callee_stack16
ret void
}
define fastcc void @caller_to8_from24([8 x i64], i64 %a, i64 %b, i64 %c) #0 {
; COMMON-LABEL: caller_to8_from24:
-; COMMON-NOT: sub sp,
+; COMMON: // %bb.0:
+; COMMON-NEXT: mov w8, #42 // =0x2a
+; COMMON-NEXT: str x8, [sp, #16]!
+; COMMON-NEXT: .cfi_def_cfa_offset -16
+; COMMON-NEXT: b callee_stack8
; Key point is that the "%a" should go where at #16 above SP on entry.
tail call fastcc void @callee_stack8([8 x i64] undef, i64 42)
ret void
-; COMMON: str {{x[0-9]+}}, [sp, #16]!
-; COMMON: .cfi_def_cfa_offset -16
-; COMMON-NEXT: b callee_stack8
}
define fastcc void @caller_to16_from16([8 x i64], i64 %a, i64 %b) #0 {
-; COMMON-LABEL: caller_to16_from16:
-; COMMON-NOT: sub sp,
-
; Here we want to make sure that both loads happen before the stores:
; otherwise either %a or %b will be wrongly clobbered.
+; SDAG-LABEL: caller_to16_from16:
+; SDAG: // %bb.0:
+; SDAG-NEXT: ldp x9, x8, [sp]
+; SDAG-NEXT: stp x8, x9, [sp]
+; SDAG-NEXT: b callee_stack16
+;
+; GISEL-LABEL: caller_to16_from16:
+; GISEL: // %bb.0:
+; GISEL-NEXT: ldp x8, x9, [sp]
+; GISEL-NEXT: stp x9, x8, [sp]
+; GISEL-NEXT: b callee_stack16
tail call fastcc void @callee_stack16([8 x i64] undef, i64 %b, i64 %a)
ret void
-; COMMON: ldp {{x[0-9]+}}, {{x[0-9]+}}, [sp]
-; COMMON: stp {{x[0-9]+}}, {{x[0-9]+}}, [sp]
-; COMMON-NEXT: b callee_stack16
}
define fastcc void @disable_tail_calls() nounwind "disable-tail-calls"="true" {
; COMMON-LABEL: disable_tail_calls:
-; COMMON-NEXT: // %bb.
+; COMMON: // %bb.0:
+; COMMON-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; COMMON-NEXT: bl callee_stack0
+; COMMON-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; COMMON-NEXT: ret
tail call fastcc void @callee_stack0()
ret void
-; COMMON: bl callee_stack0
-; COMMON: ret
}
; Weakly-referenced extern functions cannot be tail-called, as AAELF does
; not define the behaviour of branch instructions to undefined weak symbols.
define fastcc void @caller_weak() #0 {
; COMMON-LABEL: caller_weak:
-; COMMON: bl callee_weak
+; COMMON: // %bb.0:
+; COMMON-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; COMMON-NEXT: .cfi_def_cfa_offset 16
+; COMMON-NEXT: .cfi_offset w30, -16
+; COMMON-NEXT: bl callee_weak
+; COMMON-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; COMMON-NEXT: .cfi_def_cfa_offset 0
+; COMMON-NEXT: .cfi_restore w30
+; COMMON-NEXT: ret
tail call void @callee_weak()
ret void
}
@@ -118,17 +147,17 @@ define fastcc void @caller_weak() #0 {
declare { [2 x float] } @get_vec2()
define { [3 x float] } @test_add_elem() #0 {
-; SDAG-LABEL: test_add_elem:
-; SDAG: bl get_vec2
-; SDAG: fmov s2, #1.0
-; SDAG: ret
-; GISEL-LABEL: test_add_elem:
-; GISEL: str x30, [sp, #-16]!
-; GISEL: bl get_vec2
-; GISEL: fmov s2, #1.0
-; GISEL: ldr x30, [sp], #16
-; GISEL: ret
-
+; COMMON-LABEL: test_add_elem:
+; COMMON: // %bb.0:
+; COMMON-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; COMMON-NEXT: .cfi_def_cfa_offset 16
+; COMMON-NEXT: .cfi_offset w30, -16
+; COMMON-NEXT: bl get_vec2
+; COMMON-NEXT: fmov s2, #1.00000000
+; COMMON-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; COMMON-NEXT: .cfi_def_cfa_offset 0
+; COMMON-NEXT: .cfi_restore w30
+; COMMON-NEXT: ret
%call = tail call { [2 x float] } @get_vec2()
%arr = extractvalue { [2 x float] } %call, 0
%arr.0 = extractvalue [2 x float] %arr, 0
@@ -143,10 +172,28 @@ define { [3 x float] } @test_add_elem() #0 {
declare double @get_double()
define { double, [2 x double] } @test_mismatched_insert() #0 {
; COMMON-LABEL: test_mismatched_insert:
-; COMMON: bl get_double
-; COMMON: bl get_double
-; COMMON: bl get_double
-; COMMON: ret
+; COMMON: // %bb.0:
+; COMMON-NEXT: stp d9, d8, [sp, #-32]! // 16-byte Folded Spill
+; COMMON-NEXT: .cfi_def_cfa_offset 32
+; COMMON-NEXT: str x30, [sp, #16] // 8-byte Spill
+; COMMON-NEXT: .cfi_offset w30, -16
+; COMMON-NEXT: .cfi_offset b8, -24
+; COMMON-NEXT: .cfi_offset b9, -32
+; COMMON-NEXT: bl get_double
+; COMMON-NEXT: fmov d8, d0
+; COMMON-NEXT: bl get_double
+; COMMON-NEXT: fmov d9, d0
+; COMMON-NEXT: bl get_double
+; COMMON-NEXT: ldr x30, [sp, #16] // 8-byte Reload
+; COMMON-NEXT: fmov d2, d0
+; COMMON-NEXT: fmov d0, d8
+; COMMON-NEXT: fmov d1, d9
+; COMMON-NEXT: ldp d9, d8, [sp], #32 // 16-byte Folded Reload
+; COMMON-NEXT: .cfi_def_cfa_offset 0
+; COMMON-NEXT: .cfi_restore w30
+; COMMON-NEXT: .cfi_restore b8
+; COMMON-NEXT: .cfi_restore b9
+; COMMON-NEXT: ret
%val0 = call double @get_double()
%val1 = call double @get_double()
@@ -159,4 +206,4 @@ define { double, [2 x double] } @test_mismatched_insert() #0 {
ret { double, [2 x double] } %res.012
}
-attributes #0 = { uwtable }
\ No newline at end of file
+attributes #0 = { uwtable }
>From 693b12939d736e829cfdfdc411f77db957de4b54 Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Sun, 28 Jun 2026 23:16:36 +0200
Subject: [PATCH 05/11] add basic test
---
llvm/test/CodeGen/AArch64/sibcall-byval.ll | 27 ++++++++++++++++++++++
1 file changed, 27 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/sibcall-byval.ll
diff --git a/llvm/test/CodeGen/AArch64/sibcall-byval.ll b/llvm/test/CodeGen/AArch64/sibcall-byval.ll
new file mode 100644
index 0000000000000..f66080150dcff
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sibcall-byval.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu -O3 %s -verify-machineinstrs -o - | FileCheck %s --check-prefixes=CHECK-SD
+; RQN: llc -mtriple=aarch64-unknown-linux-gnu -O3 -global-isel -global-isel-abort=1 %s -verify-machineinstrs -o - | FileCheck %s --check-prefixes=CHECK-GI
+
+%struct.p = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 }
+
+define i32 @f(ptr byval(%struct.p) align 4 %q) nounwind {
+; CHECK-SD-LABEL: f:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: b g
+entry:
+ %call = tail call i32 @g(ptr byval(%struct.p) align 4 %q) nounwind
+ ret i32 %call
+}
+
+declare i32 @g(ptr byval(%struct.p) align 4)
+
+define i32 @h(ptr byval(%struct.p) align 4 %q, i32 %r) nounwind {
+; CHECK-SD-LABEL: h:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: b i
+entry:
+ %call = tail call i32 @i(ptr byval(%struct.p) align 4 %q, i32 %r) nounwind
+ ret i32 %call
+}
+
+declare i32 @i(ptr byval(%struct.p) align 4, i32)
>From cd42c40f0196db4991990eb5089ca86575176e4b Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Mon, 29 Jun 2026 00:06:30 +0200
Subject: [PATCH 06/11] remove gisel assert on byval arguments
---
.../AArch64/GISel/AArch64CallLowering.cpp | 15 +++------------
llvm/test/CodeGen/AArch64/sibcall-byval.ll | 18 +++++++++++++++++-
2 files changed, 20 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
index af88bc51e1ae7..895f0d7669bab 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
@@ -262,8 +262,6 @@ struct OutgoingArgHandler : public CallLowering::OutgoingValueHandler {
LLT s64 = LLT::integer(64);
if (IsTailCall) {
- assert(!Flags.isByVal() && "byval unhandled with tail calls");
-
Offset += FPDiff;
int FI = MF.getFrameInfo().CreateFixedObject(Size, Offset, true);
auto FIReg = MIRBuilder.buildFrameIndex(p0, FI);
@@ -1005,13 +1003,6 @@ bool AArch64CallLowering::isEligibleForTailCallOptimization(
return false;
}
- // Byval parameters hand the function a pointer directly into the stack area
- // we want to reuse during a tail call. Working around this *is* possible (see
- // X86).
- //
- // FIXME: In AArch64ISelLowering, this isn't worked around. Can/should we try
- // it?
- //
// On Windows, "inreg" attributes signify non-aggregate indirect returns.
// In this case, it is necessary to save/restore X0 in the callee. Tail
// call opt interferes with this. So we disable tail call opt when the
@@ -1023,10 +1014,10 @@ bool AArch64CallLowering::isEligibleForTailCallOptimization(
// because would have to move into the swifterror register before the
// tail call.
if (any_of(CallerF.args(), [](const Argument &A) {
- return A.hasByValAttr() || A.hasInRegAttr() || A.hasSwiftErrorAttr();
+ return A.hasInRegAttr() || A.hasSwiftErrorAttr();
})) {
- LLVM_DEBUG(dbgs() << "... Cannot tail call from callers with byval, "
- "inreg, or swifterror arguments\n");
+ LLVM_DEBUG(dbgs() << "... Cannot tail call from callers with inreg"
+ " or swifterror arguments\n");
return false;
}
diff --git a/llvm/test/CodeGen/AArch64/sibcall-byval.ll b/llvm/test/CodeGen/AArch64/sibcall-byval.ll
index f66080150dcff..0fa204320691e 100644
--- a/llvm/test/CodeGen/AArch64/sibcall-byval.ll
+++ b/llvm/test/CodeGen/AArch64/sibcall-byval.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=aarch64-unknown-linux-gnu -O3 %s -verify-machineinstrs -o - | FileCheck %s --check-prefixes=CHECK-SD
-; RQN: llc -mtriple=aarch64-unknown-linux-gnu -O3 -global-isel -global-isel-abort=1 %s -verify-machineinstrs -o - | FileCheck %s --check-prefixes=CHECK-GI
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu -O3 -global-isel -global-isel-abort=1 %s -verify-machineinstrs -o - | FileCheck %s --check-prefixes=CHECK-GI
%struct.p = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 }
@@ -8,6 +8,14 @@ define i32 @f(ptr byval(%struct.p) align 4 %q) nounwind {
; CHECK-SD-LABEL: f:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: b g
+;
+; CHECK-GI-LABEL: f:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldp q1, q0, [sp]
+; CHECK-GI-NEXT: stp q1, q0, [sp]
+; CHECK-GI-NEXT: ldr q0, [sp, #32]
+; CHECK-GI-NEXT: str q0, [sp, #32]
+; CHECK-GI-NEXT: b g
entry:
%call = tail call i32 @g(ptr byval(%struct.p) align 4 %q) nounwind
ret i32 %call
@@ -19,6 +27,14 @@ define i32 @h(ptr byval(%struct.p) align 4 %q, i32 %r) nounwind {
; CHECK-SD-LABEL: h:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: b i
+;
+; CHECK-GI-LABEL: h:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: ldp q1, q0, [sp]
+; CHECK-GI-NEXT: stp q1, q0, [sp]
+; CHECK-GI-NEXT: ldr q0, [sp, #32]
+; CHECK-GI-NEXT: str q0, [sp, #32]
+; CHECK-GI-NEXT: b i
entry:
%call = tail call i32 @i(ptr byval(%struct.p) align 4 %q, i32 %r) nounwind
ret i32 %call
>From e1d4bc79e1a394d88d9227c215dd42b379062dd5 Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Mon, 29 Jun 2026 00:26:58 +0200
Subject: [PATCH 07/11] add failing test case that needs memmove instead of
memcpy
---
llvm/test/CodeGen/AArch64/sibcall-byval.ll | 35 ++++++++++++++++++++++
1 file changed, 35 insertions(+)
diff --git a/llvm/test/CodeGen/AArch64/sibcall-byval.ll b/llvm/test/CodeGen/AArch64/sibcall-byval.ll
index 0fa204320691e..fda3a9c0c4bc6 100644
--- a/llvm/test/CodeGen/AArch64/sibcall-byval.ll
+++ b/llvm/test/CodeGen/AArch64/sibcall-byval.ll
@@ -41,3 +41,38 @@ entry:
}
declare i32 @i(ptr byval(%struct.p) align 4, i32)
+
+; The data on the stack overlaps with the space used to pass arguments to the tail call. To prevent
+; stepping on our own tail, the byval argument should pass through a temporary in the stack frame.
+define i32 @overlap_forward(i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, ptr byval(%struct.p) align 8 %q, i64 %pad) nounwind {
+; CHECK-SD-LABEL: overlap_forward:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: sub sp, sp, #48
+; CHECK-SD-NEXT: ldp q0, q1, [sp, #48]
+; CHECK-SD-NEXT: mov w8, #99 // =0x63
+; CHECK-SD-NEXT: ldr q2, [sp, #80]
+; CHECK-SD-NEXT: stp q0, q1, [sp]
+; CHECK-SD-NEXT: str q2, [sp, #32]
+; CHECK-SD-NEXT: stur q0, [sp, #56]
+; CHECK-SD-NEXT: stur q1, [sp, #72]
+; CHECK-SD-NEXT: stur q2, [sp, #88]
+; CHECK-SD-NEXT: str x8, [sp, #48]!
+; CHECK-SD-NEXT: b overlap_callee
+;
+; CHECK-GI-LABEL: overlap_forward:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: mov w8, #99 // =0x63
+; CHECK-GI-NEXT: str x8, [sp]
+; CHECK-GI-NEXT: ldr q0, [sp]
+; CHECK-GI-NEXT: stur q0, [sp, #8]
+; CHECK-GI-NEXT: ldr q0, [sp, #16]
+; CHECK-GI-NEXT: stur q0, [sp, #24]
+; CHECK-GI-NEXT: ldr q0, [sp, #32]
+; CHECK-GI-NEXT: stur q0, [sp, #40]
+; CHECK-GI-NEXT: b overlap_callee
+entry:
+ %call = tail call i32 @overlap_callee(i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 99, ptr byval(%struct.p) align 8 %q) nounwind
+ ret i32 %call
+}
+
+declare i32 @overlap_callee(i64, i64, i64, i64, i64, i64, i64, i64, i64, ptr byval(%struct.p) align 8)
>From d8d7f8328485d790d4c648a861b85b7902fd8ab9 Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Mon, 29 Jun 2026 21:16:35 +0200
Subject: [PATCH 08/11] gisel basics
---
.../llvm/CodeGen/GlobalISel/CallLowering.h | 5 ++++-
.../AArch64/GISel/AArch64CallLowering.cpp | 17 +++++++++++++++++
2 files changed, 21 insertions(+), 1 deletion(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
index 110f40a817770..2c835d3c66644 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
@@ -316,7 +316,10 @@ class LLVM_ABI CallLowering {
/// Do a memory copy of \p MemSize bytes from \p SrcPtr to \p DstPtr. This
/// is necessary for outgoing stack-passed byval arguments.
- void
+ ///
+ /// Targets may override this, e.g. to pass tail call arguments through a
+ /// temporary.
+ virtual void
copyArgumentMemory(const ArgInfo &Arg, Register DstPtr, Register SrcPtr,
const MachinePointerInfo &DstPtrInfo, Align DstAlign,
const MachinePointerInfo &SrcPtrInfo, Align SrcAlign,
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
index 895f0d7669bab..d7f1dbef44724 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
@@ -298,6 +298,23 @@ struct OutgoingArgHandler : public CallLowering::OutgoingValueHandler {
MIRBuilder.buildCopy(PhysReg, ExtReg);
}
+ enum ByValCopyKind { CopyOnce, CopyViaTemp, NoCopy };
+ ByValCopyKind classifyByValForTailCall(Register SrcPtr,
+ const CCValAssign &VA) const {
+ return CopyViaTemp;
+ }
+
+ void copyArgumentMemory(const CallLowering::ArgInfo &Arg, Register DstPtr,
+ Register SrcPtr, const MachinePointerInfo &DstPtrInfo,
+ Align DstAlign, const MachinePointerInfo &SrcPtrInfo,
+ Align SrcAlign, uint64_t MemSize,
+ CCValAssign &VA) const override {
+ // Otherwise just use the default implementation.
+ CallLowering::OutgoingValueHandler::copyArgumentMemory(
+ Arg, DstPtr, SrcPtr, DstPtrInfo, DstAlign, SrcPtrInfo, SrcAlign,
+ MemSize, VA);
+ }
+
/// Check whether a stack argument requires lowering in a tail call.
static bool shouldLowerTailCallStackArg(const MachineFunction &MF,
const CCValAssign &VA,
>From cc115c1c86efd465809f7c766daf5f020b6f1336 Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Mon, 29 Jun 2026 22:25:55 +0200
Subject: [PATCH 09/11] GISel implementation
or an attempt anyway at porting the x86 and normal aarch64 implementation
---
.../Target/AArch64/AArch64ISelLowering.cpp | 6 +-
.../AArch64/GISel/AArch64CallLowering.cpp | 64 ++++++++++++++++++-
llvm/test/CodeGen/AArch64/sibcall-byval.ll | 27 ++++----
3 files changed, 78 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 4e2c78c94d62e..3786439d96917 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -9989,7 +9989,8 @@ AArch64TargetLowering::ByValNeedsCopyForTailCall(SelectionDAG &DAG, SDValue Src,
ISD::ArgFlagsTy Flags) const {
MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
- // Globals are always safe to copy from.
+ // Globals and constant pools are not a part of the stack frame, hence
+ // always safe to copy from.
if (isa<GlobalAddressSDNode>(Src) || isa<ExternalSymbolSDNode>(Src))
return CopyOnce;
@@ -10010,8 +10011,7 @@ AArch64TargetLowering::ByValNeedsCopyForTailCall(SelectionDAG &DAG, SDValue Src,
// If the source is in the local frame, then the copy to the argument memory
// is always valid.
- bool FixedSrc = MFI.isFixedObjectIndex(SrcFI);
- if (!FixedSrc || (FixedSrc && SrcOffset < 0))
+ if (!MFI.isFixedObjectIndex(SrcFI) || SrcOffset < 0)
return CopyOnce;
// If the value is already in the correct location, then no copying is
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
index d7f1dbef44724..150d39cf70317 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
@@ -300,7 +300,44 @@ struct OutgoingArgHandler : public CallLowering::OutgoingValueHandler {
enum ByValCopyKind { CopyOnce, CopyViaTemp, NoCopy };
ByValCopyKind classifyByValForTailCall(Register SrcPtr,
- const CCValAssign &VA) const {
+ Register DstPtr) const {
+ const MachineFunction &MF = MIRBuilder.getMF();
+ const MachineFrameInfo &MFI = MF.getFrameInfo();
+
+ // Find the defining instruction (looking through copies).
+ MachineInstr *SrcDef = getDefIgnoringCopies(SrcPtr, MRI);
+ MachineInstr *DstDef = getDefIgnoringCopies(DstPtr, MRI);
+
+ // Convervatively copy when we can't find a root.
+ if (!SrcDef || !DstDef)
+ return CopyViaTemp;
+
+ // Globals are always safe to copy from.
+ if (SrcDef->getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
+ SrcDef->getOpcode() == TargetOpcode::G_CONSTANT_POOL)
+ return CopyOnce;
+
+ // Can only analyse frame index nodes, conservatively assume we need a
+ // temporary.
+ if (SrcDef->getOpcode() != TargetOpcode::G_FRAME_INDEX ||
+ DstDef->getOpcode() != TargetOpcode::G_FRAME_INDEX)
+ return CopyViaTemp;
+
+ int SrcFI = SrcDef->getOperand(1).getIndex();
+ int64_t SrcOffset = MFI.getObjectOffset(SrcFI);
+
+ // If the source is in the local frame, then the copy to the argument memory
+ // is always valid.
+ if (!MFI.isFixedObjectIndex(SrcFI) || SrcOffset < 0)
+ return CopyOnce;
+
+ // If the value is already in the correct location, then no copying is
+ // needed. If not, then we need to copy via a temporary.
+ int DstFI = DstDef->getOperand(1).getIndex();
+ int64_t DstOffset = MFI.getObjectOffset(DstFI);
+ if (SrcOffset == DstOffset)
+ return NoCopy;
+
return CopyViaTemp;
}
@@ -309,6 +346,31 @@ struct OutgoingArgHandler : public CallLowering::OutgoingValueHandler {
Align DstAlign, const MachinePointerInfo &SrcPtrInfo,
Align SrcAlign, uint64_t MemSize,
CCValAssign &VA) const override {
+ if (IsTailCall) {
+ ByValCopyKind Copy = classifyByValForTailCall(SrcPtr, DstPtr);
+ if (Copy == NoCopy) {
+ // The value is already in the right place.
+ return;
+ } else if (Copy == CopyViaTemp) {
+ // Copy first to a local, then to the destination.
+ MachineFunction &MF = MIRBuilder.getMF();
+ int TempFI = MF.getFrameInfo().CreateStackObject(
+ MemSize, std::max(SrcAlign, DstAlign), /*isSpillSlot=*/false);
+ LLT p0 = LLT::pointer(0, 64);
+ Register Temp = MIRBuilder.buildFrameIndex(p0, TempFI).getReg(0);
+ MachinePointerInfo TempMPO =
+ MachinePointerInfo::getFixedStack(MF, TempFI);
+
+ CallLowering::OutgoingValueHandler::copyArgumentMemory(
+ Arg, Temp, SrcPtr, TempMPO, DstAlign, SrcPtrInfo, SrcAlign, MemSize,
+ VA);
+ CallLowering::OutgoingValueHandler::copyArgumentMemory(
+ Arg, DstPtr, Temp, DstPtrInfo, DstAlign, TempMPO, DstAlign, MemSize,
+ VA);
+ return;
+ }
+ }
+
// Otherwise just use the default implementation.
CallLowering::OutgoingValueHandler::copyArgumentMemory(
Arg, DstPtr, SrcPtr, DstPtrInfo, DstAlign, SrcPtrInfo, SrcAlign,
diff --git a/llvm/test/CodeGen/AArch64/sibcall-byval.ll b/llvm/test/CodeGen/AArch64/sibcall-byval.ll
index fda3a9c0c4bc6..2868b586b90d9 100644
--- a/llvm/test/CodeGen/AArch64/sibcall-byval.ll
+++ b/llvm/test/CodeGen/AArch64/sibcall-byval.ll
@@ -11,10 +11,6 @@ define i32 @f(ptr byval(%struct.p) align 4 %q) nounwind {
;
; CHECK-GI-LABEL: f:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: ldp q1, q0, [sp]
-; CHECK-GI-NEXT: stp q1, q0, [sp]
-; CHECK-GI-NEXT: ldr q0, [sp, #32]
-; CHECK-GI-NEXT: str q0, [sp, #32]
; CHECK-GI-NEXT: b g
entry:
%call = tail call i32 @g(ptr byval(%struct.p) align 4 %q) nounwind
@@ -30,10 +26,6 @@ define i32 @h(ptr byval(%struct.p) align 4 %q, i32 %r) nounwind {
;
; CHECK-GI-LABEL: h:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: ldp q1, q0, [sp]
-; CHECK-GI-NEXT: stp q1, q0, [sp]
-; CHECK-GI-NEXT: ldr q0, [sp, #32]
-; CHECK-GI-NEXT: str q0, [sp, #32]
; CHECK-GI-NEXT: b i
entry:
%call = tail call i32 @i(ptr byval(%struct.p) align 4 %q, i32 %r) nounwind
@@ -61,14 +53,19 @@ define i32 @overlap_forward(i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5
;
; CHECK-GI-LABEL: overlap_forward:
; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: sub sp, sp, #48
; CHECK-GI-NEXT: mov w8, #99 // =0x63
-; CHECK-GI-NEXT: str x8, [sp]
-; CHECK-GI-NEXT: ldr q0, [sp]
-; CHECK-GI-NEXT: stur q0, [sp, #8]
-; CHECK-GI-NEXT: ldr q0, [sp, #16]
-; CHECK-GI-NEXT: stur q0, [sp, #24]
-; CHECK-GI-NEXT: ldr q0, [sp, #32]
-; CHECK-GI-NEXT: stur q0, [sp, #40]
+; CHECK-GI-NEXT: str x8, [sp, #48]
+; CHECK-GI-NEXT: ldp q1, q0, [sp, #48]
+; CHECK-GI-NEXT: stp q1, q0, [sp]
+; CHECK-GI-NEXT: ldr q0, [sp, #80]
+; CHECK-GI-NEXT: ldr q1, [sp]
+; CHECK-GI-NEXT: str q0, [sp, #32]
+; CHECK-GI-NEXT: stur q1, [sp, #56]
+; CHECK-GI-NEXT: ldp q1, q0, [sp, #16]
+; CHECK-GI-NEXT: stur q1, [sp, #72]
+; CHECK-GI-NEXT: stur q0, [sp, #88]
+; CHECK-GI-NEXT: add sp, sp, #48
; CHECK-GI-NEXT: b overlap_callee
entry:
%call = tail call i32 @overlap_callee(i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 99, ptr byval(%struct.p) align 8 %q) nounwind
>From 49ef6db2d008b5d5eab25ca4b27ed0c6bc8c29b7 Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Tue, 30 Jun 2026 11:01:14 +0200
Subject: [PATCH 10/11] add tailcc byval test
---
llvm/test/CodeGen/AArch64/tailcc-tail-call.ll | 46 ++++++++++++++++++-
1 file changed, 45 insertions(+), 1 deletion(-)
diff --git a/llvm/test/CodeGen/AArch64/tailcc-tail-call.ll b/llvm/test/CodeGen/AArch64/tailcc-tail-call.ll
index a1482ac5d53cb..d74948928bab1 100644
--- a/llvm/test/CodeGen/AArch64/tailcc-tail-call.ll
+++ b/llvm/test/CodeGen/AArch64/tailcc-tail-call.ll
@@ -229,4 +229,48 @@ define tailcc void @fromtail_toC() #0 {
ret void
}
-attributes #0 = { uwtable }
\ No newline at end of file
+declare tailcc i32 @all_registers_callee(i32 %p1, i32 %p2, i32 %p3, i32 %p4, i32 %p5, i32 %p6, i32 %p7, i32 %p8, i32 %a, i32 %b)
+
+define tailcc i32 @all_registers_caller(i32 %p1, i32 %p2, i32 %p3, i32 %p4, i32 %p5, i32 %p6, i32 %p7, i32 %p8, i32 %in1, i32 %in2) {
+; COMMON-LABEL: all_registers_caller:
+; COMMON: // %bb.0: // %entry
+; COMMON-NEXT: ldr w8, [sp]
+; COMMON-NEXT: ldr w9, [sp, #8]
+; COMMON-NEXT: add w8, w8, w0
+; COMMON-NEXT: str w9, [sp]
+; COMMON-NEXT: str w8, [sp, #8]
+; COMMON-NEXT: b all_registers_callee
+entry:
+ %tmp = add i32 %in1, %p1
+ %retval = tail call tailcc i32 @all_registers_callee(i32 %p1, i32 %p2, i32 %p3, i32 %p4, i32 %p5, i32 %p6, i32 %p7, i32 %p8, i32 %in2, i32 %tmp)
+ ret i32 %retval
+}
+
+define tailcc noundef i64 @call_with_byval_caller(i64 noundef %a, i64 noundef %d) {
+; COMMON-LABEL: call_with_byval_caller:
+; COMMON: // %bb.0: // %start
+; COMMON-NEXT: mov x8, #-4919131752989213765 // =0xbbbbbbbbbbbbbbbb
+; COMMON-NEXT: stp x0, x8, [sp, #-64]!
+; COMMON-NEXT: .cfi_def_cfa_offset 64
+; COMMON-NEXT: mov x9, #-3689348814741910324 // =0xcccccccccccccccc
+; COMMON-NEXT: stp x9, x1, [sp, #16]
+; COMMON-NEXT: ldp q1, q0, [sp]
+; COMMON-NEXT: stp q1, q0, [sp, #32]!
+; COMMON-NEXT: b call_with_byval_callee
+start:
+ %large = alloca [4 x i64], align 8
+ call void @llvm.lifetime.start.p0(ptr nonnull %large)
+ store i64 %a, ptr %large, align 8
+ %0 = getelementptr inbounds nuw i8, ptr %large, i64 8
+ store i64 -4919131752989213765, ptr %0, align 8
+ %1 = getelementptr inbounds nuw i8, ptr %large, i64 16
+ store i64 -3689348814741910324, ptr %1, align 8
+ %2 = getelementptr inbounds nuw i8, ptr %large, i64 24
+ store i64 %d, ptr %2, align 8
+ %3 = musttail call tailcc i64 @call_with_byval_callee(ptr byval([4 x i64]) %large)
+ ret i64 %3
+}
+
+declare tailcc noundef i64 @call_with_byval_callee(ptr byval([4 x i64]) %large)
+
+attributes #0 = { uwtable }
>From 8a7d4a2d8aeda4176c0b05fc8ac5c5afa40fb965 Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Tue, 30 Jun 2026 14:15:36 +0200
Subject: [PATCH 11/11] more tests
---
llvm/test/CodeGen/AArch64/tailcc-tail-call.ll | 111 ++++++++++++++++++
1 file changed, 111 insertions(+)
diff --git a/llvm/test/CodeGen/AArch64/tailcc-tail-call.ll b/llvm/test/CodeGen/AArch64/tailcc-tail-call.ll
index d74948928bab1..29a4cbb9c1912 100644
--- a/llvm/test/CodeGen/AArch64/tailcc-tail-call.ll
+++ b/llvm/test/CodeGen/AArch64/tailcc-tail-call.ll
@@ -273,4 +273,115 @@ start:
declare tailcc noundef i64 @call_with_byval_callee(ptr byval([4 x i64]) %large)
+ at array_32xi8 = constant [32 x i8] c"\01\00\00\00\00\00\00\00\02\00\00\00\00\00\00\00\03\00\00\00\00\00\00\00\04\00\00\00\00\00\00\00"
+define tailcc i64 @from_global() {
+; COMMON-LABEL: from_global:
+; COMMON: // %bb.0:
+; COMMON-NEXT: adrp x8, :got:array_32xi8
+; COMMON-NEXT: ldr x8, [x8, :got_lo12:array_32xi8]
+; COMMON-NEXT: ldr q0, [x8]
+; COMMON-NEXT: str q0, [sp, #-32]!
+; COMMON-NEXT: .cfi_def_cfa_offset 32
+; COMMON-NEXT: ldr q0, [x8, #16]
+; COMMON-NEXT: str q0, [sp, #16]
+; COMMON-NEXT: b callee_byval_32xi8
+ %r = musttail call tailcc i64 @callee_byval_32xi8(ptr byval([32 x i8]) align 8 @array_32xi8)
+ ret i64 %r
+}
+
+declare tailcc i64 @callee_byval_32xi8(ptr byval([32 x i8]) align 8)
+
+define tailcc i64 @forward_incoming(ptr byval([32 x i8]) align 8 %p) {
+; COMMON-LABEL: forward_incoming:
+; COMMON: // %bb.0:
+; COMMON-NEXT: b callee_byval_32xi8
+ %r = tail call tailcc i64 @callee_byval_32xi8(ptr byval([32 x i8]) align 8 %p)
+ ret i64 %r
+}
+
+define tailcc i64 @swap_incoming(ptr byval([32 x i8]) align 8 %p, ptr byval([32 x i8]) align 8 %q) {
+; SDAG-LABEL: swap_incoming:
+; SDAG: // %bb.0:
+; SDAG-NEXT: sub sp, sp, #64
+; SDAG-NEXT: .cfi_def_cfa_offset 64
+; SDAG-NEXT: ldp q0, q1, [sp, #96]
+; SDAG-NEXT: ldp q2, q3, [sp, #64]
+; SDAG-NEXT: stp q0, q1, [sp, #32]
+; SDAG-NEXT: stp q2, q3, [sp]
+; SDAG-NEXT: stp q0, q1, [sp, #64]
+; SDAG-NEXT: stp q2, q3, [sp, #96]
+; SDAG-NEXT: add sp, sp, #64
+; SDAG-NEXT: b swap_incoming_callee
+;
+; GISEL-LABEL: swap_incoming:
+; GISEL: // %bb.0:
+; GISEL-NEXT: sub sp, sp, #64
+; GISEL-NEXT: .cfi_def_cfa_offset 64
+; GISEL-NEXT: ldp q1, q0, [sp, #96]
+; GISEL-NEXT: stp q1, q0, [sp, #32]
+; GISEL-NEXT: ldp q1, q0, [sp, #32]
+; GISEL-NEXT: stp q1, q0, [sp, #64]
+; GISEL-NEXT: ldp q1, q0, [sp, #64]
+; GISEL-NEXT: stp q1, q0, [sp]
+; GISEL-NEXT: ldp q1, q0, [sp]
+; GISEL-NEXT: stp q1, q0, [sp, #96]
+; GISEL-NEXT: add sp, sp, #64
+; GISEL-NEXT: b swap_incoming_callee
+ %r = tail call tailcc i64 @swap_incoming_callee(ptr byval([32 x i8]) align 8 %q, ptr byval([32 x i8]) align 8 %p)
+ ret i64 %r
+}
+
+declare tailcc i64 @swap_incoming_callee(ptr byval([32 x i8]) align 8, ptr byval([32 x i8]) align 8)
+
+
+define tailcc noundef i64 @swap_local_byval(i64 noundef %a, i64 noundef %d, ptr byval([32 x i8]) align 8 %p) {
+; COMMON-LABEL: call_with_byval_caller:
+; COMMON: // %bb.0: // %start
+; COMMON-NEXT: mov x8, #-4919131752989213765 // =0xbbbbbbbbbbbbbbbb
+; COMMON-NEXT: stp x0, x8, [sp, #-64]!
+; COMMON-NEXT: .cfi_def_cfa_offset 64
+; COMMON-NEXT: mov x9, #-3689348814741910324 // =0xcccccccccccccccc
+; COMMON-NEXT: stp x9, x1, [sp, #16]
+; COMMON-NEXT: ldp q1, q0, [sp]
+; COMMON-NEXT: stp q1, q0, [sp, #32]!
+; COMMON-NEXT: b call_with_byval_callee
+start:
+ %large = alloca [32 x i8], align 8
+ call void @llvm.lifetime.start.p0(ptr nonnull %large)
+ store i64 %a, ptr %large, align 8
+ %0 = getelementptr inbounds nuw i8, ptr %large, i64 8
+ store i64 -4919131752989213765, ptr %0, align 8
+ %1 = getelementptr inbounds nuw i8, ptr %large, i64 16
+ store i64 -3689348814741910324, ptr %1, align 8
+ %2 = getelementptr inbounds nuw i8, ptr %large, i64 24
+ store i64 %d, ptr %2, align 8
+ %3 = musttail call tailcc i64 @swap_incoming_callee(ptr byval([32 x i8]) align 8 %large, ptr byval([32 x i8]) align 8 %p)
+ ret i64 %3
+}
+
+declare tailcc i64 @overlap_callee(i64, i64, i64, i64, i64, i64, i64, i64, i64, ptr byval([64 x i8]) align 8)
+
+define tailcc i64 @forward_overlap(i64 %r0, i64 %r1, i64 %r2, i64 %r3, i64 %r4, i64 %r5, i64 %r6, i64 %r7, ptr byval([64 x i8]) align 8 %p) {
+; COMMON-LABEL: forward_overlap:
+; COMMON: // %bb.0:
+; COMMON-NEXT: sub sp, sp, #80
+; COMMON-NEXT: .cfi_def_cfa_offset 80
+; COMMON-NEXT: ldp q1, q0, [sp, #80]
+; COMMON-NEXT: mov w8, #99 // =0x63
+; COMMON-NEXT: str x8, [sp, #64]
+; COMMON-NEXT: stp q1, q0, [sp]
+; COMMON-NEXT: ldp q1, q0, [sp, #112]
+; COMMON-NEXT: stp q1, q0, [sp, #32]
+; COMMON-NEXT: ldp q1, q0, [sp]
+; COMMON-NEXT: stur q1, [sp, #72]
+; COMMON-NEXT: stur q0, [sp, #88]
+; COMMON-NEXT: ldp q1, q0, [sp, #32]
+; COMMON-NEXT: stur q1, [sp, #104]
+; COMMON-NEXT: stur q0, [sp, #120]
+; COMMON-NEXT: add sp, sp, #64
+; COMMON-NEXT: b overlap_callee
+ %r = musttail call tailcc i64 @overlap_callee(i64 %r0, i64 %r1, i64 %r2, i64 %r3, i64 %r4, i64 %r5, i64 %r6, i64 %r7, i64 99, ptr byval([64 x i8]) align 8 %p)
+ ret i64 %r
+}
+
attributes #0 = { uwtable }
More information about the llvm-commits
mailing list