[llvm-branch-commits] [llvm] release/23.x: [llvm][AArch64] Ensure stack alignment in non-sibcall tail calls with FPDiff (#217156) (PR #220313)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Sep 1 10:46:13 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: llvmbot
<details>
<summary>Changes</summary>
Backport 234ce03692ede13ffd2fcb35570d801c1e332814
Requested by: @<!-- -->jroelofs
---
Full diff: https://github.com/llvm/llvm-project/pull/220313.diff
2 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+18-10)
- (added) llvm/test/CodeGen/AArch64/sme-za-tailcall-fpdiff-align.ll (+79)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index d134e7f911f83..64ec28ebf3e36 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -9447,13 +9447,15 @@ SDValue AArch64TargetLowering::LowerFormalArguments(
unsigned StackArgSize = CCInfo.getStackSize();
bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
if (DoesCalleeRestoreStack(CallConv, TailCallOpt)) {
+ const Align StackAlign = Subtarget->getFrameLowering()->getStackAlign();
+
// This is a non-standard ABI so by fiat I say we're allowed to make full
- // use of the stack area to be popped, which must be aligned to 16 bytes in
- // any case:
- StackArgSize = alignTo(StackArgSize, 16);
+ // use of the stack area to be popped, which must be aligned to the stack
+ // alignment in any case:
+ StackArgSize = alignTo(StackArgSize, StackAlign);
// If we're expected to restore the stack (e.g. fastcc) then we'll be adding
- // a multiple of 16.
+ // a multiple of the stack alignment.
FuncInfo->setArgumentStackToRestore(StackArgSize);
// This realignment carries over to the available bytes below. Our own
@@ -10286,19 +10288,24 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
// caller will deallocate the entire stack and the callee still expects its
// arguments to begin at SP+0. Completely unused for non-tail calls.
int FPDiff = 0;
+ const Align StackAlign = Subtarget->getFrameLowering()->getStackAlign();
if (IsTailCall && !IsSibCall) {
unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea();
- // Since callee will pop argument stack as a tail call, we must keep the
- // popped size 16-byte aligned.
- NumBytes = alignTo(NumBytes, 16);
-
// FPDiff will be negative if this tail call requires more space than we
// would automatically have in our incoming argument space. Positive if we
// can actually shrink the stack.
FPDiff = NumReusableBytes - NumBytes;
+ // Since callee will pop the argument stack as a tail call, we must keep the
+ // popped size aligned to the stack alignment. Either or both of NumBytes
+ // and NumReusableBytes may not have been aligned, so we further increase by
+ // the amount needed to keep FPDiff aligned, and therefore preserve the
+ // required alignment going into the callee.
+ FPDiff -= offsetToAlignment(FPDiff, StackAlign);
+ NumBytes += offsetToAlignment(FPDiff, StackAlign);
+
// Update the required reserved area if this is the tail call requiring the
// most argument stack space.
if (FPDiff < 0 && FuncInfo->getTailCallReservedStack() < (unsigned)-FPDiff)
@@ -10856,8 +10863,9 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
MF.getFunction().getParent()->getModuleFlag("import-call-optimization"))
DAG.addCalledGlobal(Chain.getNode(), CalledGlobal, OpFlags);
- uint64_t CalleePopBytes =
- DoesCalleeRestoreStack(CallConv, TailCallOpt) ? alignTo(NumBytes, 16) : 0;
+ uint64_t CalleePopBytes = DoesCalleeRestoreStack(CallConv, TailCallOpt)
+ ? alignTo(NumBytes, StackAlign)
+ : 0;
Chain = DAG.getCALLSEQ_END(Chain, NumBytes, CalleePopBytes, InGlue, DL);
InGlue = Chain.getValue(1);
diff --git a/llvm/test/CodeGen/AArch64/sme-za-tailcall-fpdiff-align.ll b/llvm/test/CodeGen/AArch64/sme-za-tailcall-fpdiff-align.ll
new file mode 100644
index 0000000000000..9c449b12f2127
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme-za-tailcall-fpdiff-align.ll
@@ -0,0 +1,79 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=arm64-apple-ios -mattr=+sme -o - | FileCheck %s
+; RUN: llc < %s -mtriple=arm64-apple-ios -mattr=+sme -global-isel-abort=2 -o - 2>&1 | FileCheck %s --check-prefix=CHECK,GISEL
+
+; GlobalISel doesn't implement SME yet, but once it does, we should be mindful of this case.
+; GISEL: warning: Instruction selection used fallback path for caller_more_args
+; GISEL: warning: Instruction selection used fallback path for caller_same_args
+
+declare void @external_func() "aarch64_inout_za"
+
+; ZA state being live for both caller and callee forces the non-sibcall tail
+; call fast path. FPDiff = (caller's incoming argument stack) - (callee's
+; outgoing argument stack) can come out either positive or negative, and in
+; either case its magnitude must be rounded away from zero to a multiple of
+; 16 to keep the stack pointer 16-byte aligned at all times.
+declare void @callee_fewer_args(i64, i64, i64, i64, i64, i64, i64, i64) "aarch64_inout_za"
+define void @caller_more_args(i64, i64, i64, i64, i64, i64, i64, i64, i64, i64) "aarch64_inout_za" uwtable {
+; CHECK-LABEL: caller_more_args:
+; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: .cfi_offset w30, -8
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: bl _external_func
+; CHECK-NEXT: mov w0, #1 ; =0x1
+; CHECK-NEXT: mov w1, #2 ; =0x2
+; CHECK-NEXT: mov w2, #3 ; =0x3
+; CHECK-NEXT: mov w3, #4 ; =0x4
+; CHECK-NEXT: mov w4, #5 ; =0x5
+; CHECK-NEXT: mov w5, #6 ; =0x6
+; CHECK-NEXT: mov w6, #7 ; =0x7
+; CHECK-NEXT: mov w7, #8 ; =0x8
+; CHECK-NEXT: ldp x29, x30, [sp], #16 ; 16-byte Folded Reload
+; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: add sp, sp, #16
+; CHECK-NEXT: .cfi_def_cfa_offset -16
+; CHECK-NEXT: .cfi_restore w30
+; CHECK-NEXT: .cfi_restore w29
+; CHECK-NEXT: b _callee_fewer_args
+entry:
+ tail call void @external_func()
+ tail call void @callee_fewer_args(i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8) "aarch64_inout_za"
+ ret void
+}
+
+; Likewise for a callee that has the same amount of argument stack as the
+; caller, we must round up the amount of used stack space in order to keep
+; FPDiff a multiple of 16, thus keeping the stack aligned for the callee.
+declare void @callee_same_args(i64, i64, i64, i64, i64, i64, i64, i64, i64) "aarch64_inout_za"
+define void @caller_same_args(i64, i64, i64, i64, i64, i64, i64, i64, i64) "aarch64_inout_za" uwtable {
+; CHECK-LABEL: caller_same_args:
+; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: stp x29, x30, [sp, #-16]! ; 16-byte Folded Spill
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: .cfi_offset w30, -8
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: bl _external_func
+; CHECK-NEXT: mov w8, #9 ; =0x9
+; CHECK-NEXT: mov w0, #1 ; =0x1
+; CHECK-NEXT: mov w1, #2 ; =0x2
+; CHECK-NEXT: str x8, [sp, #16]
+; CHECK-NEXT: mov w2, #3 ; =0x3
+; CHECK-NEXT: mov w3, #4 ; =0x4
+; CHECK-NEXT: mov w4, #5 ; =0x5
+; CHECK-NEXT: mov w5, #6 ; =0x6
+; CHECK-NEXT: mov w6, #7 ; =0x7
+; CHECK-NEXT: mov w7, #8 ; =0x8
+; CHECK-NEXT: ldp x29, x30, [sp], #16 ; 16-byte Folded Reload
+; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: .cfi_restore w30
+; CHECK-NEXT: .cfi_restore w29
+; CHECK-NEXT: b _callee_same_args
+entry:
+ tail call void @external_func()
+ tail call void @callee_same_args(i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9) "aarch64_inout_za"
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GISEL: {{.*}}
``````````
</details>
https://github.com/llvm/llvm-project/pull/220313
More information about the llvm-branch-commits
mailing list