[llvm] 3ff3c15 - [llvm][AArch64] Fix callee-pops epilogue AUT (#211889)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 10 13:13:46 PDT 2026
Author: Jon Roelofs
Date: 2026-08-10T13:13:41-07:00
New Revision: 3ff3c15845129f59030789c1e5aa3aa021a5e82f
URL: https://github.com/llvm/llvm-project/commit/3ff3c15845129f59030789c1e5aa3aa021a5e82f
DIFF: https://github.com/llvm/llvm-project/commit/3ff3c15845129f59030789c1e5aa3aa021a5e82f.diff
LOG: [llvm][AArch64] Fix callee-pops epilogue AUT (#211889)
When we combine local frame teardown with callee popped argument
cleanup, we need to pay special attention to what the SP was on entry so
that it can be used as the discriminator when AUT-ing a saved LR.
Refactors AArch64PointerAuthImpl::authenticateLR a bit to clean up the
logic that detects when we can either fold the AUT into the terminator
forming a RETA[AB][PC], or when we can hard-code the SP discriminator
into the AUT as in AUTI[AB]SP[PC].
Added:
llvm/test/CodeGen/AArch64/ptrauth-tail-call-stackadjust.ll
Modified:
llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp b/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
index 61d7f550836c4..f7eb3a9982faf 100644
--- a/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
@@ -13,6 +13,7 @@
#include "AArch64InstrInfo.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64Subtarget.h"
+#include "MCTargetDesc/AArch64AddressingModes.h"
#include "llvm/CodeGen/CFIInstBuilder.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -303,114 +304,11 @@ void AArch64PointerAuthImpl::authenticateLR(
// are placed between MBBI and TI.
MachineBasicBlock::iterator TI = MBB.getFirstInstrTerminator();
- // The AUTIASP instruction assembles to a hint instruction before v8.3a so
- // this instruction can safely used for any v8a architecture.
- // From v8.3a onwards there are optimised authenticate LR and return
- // instructions, namely RETA{A,B}, that can be used instead. In this case the
- // DW_CFA_AARCH64_negate_ra_state can't be emitted.
- bool TerminatorIsCombinable =
- TI != MBB.end() && TI->getOpcode() == AArch64::RET;
MCSymbol *PACSym = MFnI->getSigningInstrLabel();
-
- if (Subtarget->hasPAuth() && TerminatorIsCombinable && !NeedsWinCFI &&
- !MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) {
- if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
- assert(PACSym && "No PAC instruction to refer to");
- BuildMI(MBB, TI, DL,
- TII->get(UseBKey ? AArch64::RETABSPPCi : AArch64::RETAASPPCi))
- .addSym(PACSym)
- .copyImplicitOps(*MBBI)
- .setMIFlag(MachineInstr::FrameDestroy);
- } else {
- if (MFnI->branchProtectionPAuthLR()) {
- emitEpiloguePACSymOffsetIntoReg(*TII, MBB, MBBI, DL, PACSym,
- AArch64::X16);
- BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACM))
- .setMIFlag(MachineInstr::FrameDestroy);
- }
- BuildMI(MBB, TI, DL, TII->get(UseBKey ? AArch64::RETAB : AArch64::RETAA))
- .copyImplicitOps(*MBBI)
- .setMIFlag(MachineInstr::FrameDestroy);
- }
- MBB.erase(TI);
- return;
- }
-
auto &AFL = *static_cast<const AArch64FrameLowering *>(
MF.getSubtarget().getFrameLowering());
int64_t ArgumentStackToRestore = AFL.getArgumentStackToRestore(MF, MBB);
- // When ArgumentStackToRestore < 0, the tail callee pops more argument space
- // than this function received, so after the frame teardown SP is below the
- // entry SP used as the signing modifier. Reconstruct entry SP in x16 and
- // authenticate using AUTI[AB]1716 (x17=LR, x16=entry_SP).
- if (ArgumentStackToRestore < 0) {
- emitFrameOffset(MBB, MBBI, DL, AArch64::X16, AArch64::SP,
- StackOffset::getFixed(-ArgumentStackToRestore), TII,
- MachineInstr::FrameDestroy);
-
- auto emitMOV = [&](Register Dst, Register Src) {
- BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrs), Dst)
- .addReg(AArch64::XZR)
- .addReg(Src)
- .addImm(0)
- .setMIFlag(MachineInstr::FrameDestroy);
- };
-
- if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
- emitMOV(AArch64::X17, AArch64::LR);
-
- assert(PACSym && "No PAC instruction to refer to");
- emitEpiloguePACSymOffsetIntoReg(*TII, MBB, MBBI, DL, PACSym,
- AArch64::X15);
-
- unsigned AutOpc = UseBKey ? AArch64::AUTIB171615 : AArch64::AUTIA171615;
- BuildMI(MBB, MBBI, DL, TII->get(AutOpc))
- .setMIFlag(MachineInstr::FrameDestroy);
- emitAUTCFI(MBB, MBBI, EmitAsyncCFI);
-
- emitMOV(AArch64::LR, AArch64::X17);
- } else if (MFnI->branchProtectionPAuthLR()) {
- emitMOV(AArch64::X17, AArch64::LR);
-
- assert(PACSym && "No PAC instruction to refer to");
- emitEpiloguePACSymOffsetIntoReg(*TII, MBB, MBBI, DL, PACSym,
- AArch64::X15);
-
- // The PACM hint-space instruction modifies the following AUTI[AB]1716
- // to optionally take x15 as an extra operand depending on the
- // presence of +pauth-lr at runtime. On machines without +pauth-lr, it
- // behaves as a nop, and the address of the PACI[AB]SP in x15 is
- // ignored.
- BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACM))
- .setMIFlag(MachineInstr::FrameDestroy);
-
- unsigned AutOpc = UseBKey ? AArch64::AUTIB1716 : AArch64::AUTIA1716;
- BuildMI(MBB, MBBI, DL, TII->get(AutOpc))
- .setMIFlag(MachineInstr::FrameDestroy);
- emitAUTCFI(MBB, MBBI, EmitAsyncCFI);
-
- emitMOV(AArch64::LR, AArch64::X17);
- } else if (Subtarget->hasPAuth()) {
- BuildMI(MBB, MBBI, DL,
- TII->get(UseBKey ? AArch64::AUTIB : AArch64::AUTIA), AArch64::LR)
- .addUse(AArch64::LR)
- .addUse(AArch64::X16)
- .setMIFlag(MachineInstr::FrameDestroy);
- emitAUTCFI(MBB, MBBI, EmitAsyncCFI);
- } else {
- emitMOV(AArch64::X17, AArch64::LR);
-
- unsigned AutOpc = UseBKey ? AArch64::AUTIB1716 : AArch64::AUTIA1716;
- BuildMI(MBB, MBBI, DL, TII->get(AutOpc))
- .setMIFlag(MachineInstr::FrameDestroy);
- emitAUTCFI(MBB, MBBI, EmitAsyncCFI);
-
- emitMOV(AArch64::LR, AArch64::X17);
- }
- return;
- }
-
// When ArgumentStackToRestore > 0, this function received more argument
// space than the tail callee pops. The epilogue contains an SP adjustment
// (e.g. "add sp, sp, #N") to discard the leftover argument space. We must
@@ -418,41 +316,172 @@ void AArch64PointerAuthImpl::authenticateLR(
// SP discriminator. Move any such SP-adjusting instructions to after the
// authentication instruction.
//
+ // When ArgumentStackToRestore < 0, the tail callee pops more argument space
+ // than this function received, so after the frame teardown, SP is below the
+ // entry SP used as the signing modifier.
+ //
// We cannot simply bump SP first and then use AUTI[AB]SP with the bumped
// value, because the live arguments would fall below SP and potentially
- // outside the red-zone.
+ // outside the red-zone. Collect those SP adjustments in case we need to move
+ // them after the AUT.
+ int64_t Offset = -ArgumentStackToRestore;
SmallVector<MachineInstr *, 2> SPMods;
if (ArgumentStackToRestore > 0) {
for (auto I = MBBI; I->getFlag(MachineInstr::FrameDestroy); --I) {
if ((I->getOpcode() == AArch64::ADDXri ||
I->getOpcode() == AArch64::SUBXri) &&
I->getOperand(0).getReg() == AArch64::SP &&
- I->getOperand(1).getReg() == AArch64::SP)
+ I->getOperand(1).getReg() == AArch64::SP) {
SPMods.push_back(&*I);
+ int64_t Imm = I->getOperand(2).getImm()
+ << AArch64_AM::getShiftValue(I->getOperand(3).getImm());
+ Offset += I->getOpcode() == AArch64::ADDXri ? Imm : -Imm;
+ }
}
}
+
+ // If there will not be an SP bump afterward, we can use an AUT or RET form
+ // with a hardcoded SP discriminator.
+ if (!Offset) {
+ // The AUTIASP instruction assembles to a hint instruction before v8.3a so
+ // this instruction can safely be used for any v8a architecture.
+ // From v8.3a onwards there are optimised authenticate LR and return
+ // instructions, namely RETA{A,B}, that can be used instead. In this case
+ // the DW_CFA_AARCH64_negate_ra_state can't be emitted. Additionally,
+ // RET{A,B} requires the SP to match its incoming value on entry to the
+ // function.
+ bool TerminatorIsCombinable = TI != MBB.end() &&
+ TI->getOpcode() == AArch64::RET &&
+ ArgumentStackToRestore == 0;
+
+ if (Subtarget->hasPAuth() && TerminatorIsCombinable && !NeedsWinCFI &&
+ !MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) {
+ if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
+ assert(PACSym && "No PAC instruction to refer to");
+ BuildMI(MBB, TI, DL,
+ TII->get(UseBKey ? AArch64::RETABSPPCi : AArch64::RETAASPPCi))
+ .addSym(PACSym)
+ .copyImplicitOps(*MBBI)
+ .setMIFlag(MachineInstr::FrameDestroy);
+ } else {
+ if (MFnI->branchProtectionPAuthLR()) {
+ emitEpiloguePACSymOffsetIntoReg(*TII, MBB, MBBI, DL, PACSym,
+ AArch64::X16);
+ BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACM))
+ .setMIFlag(MachineInstr::FrameDestroy);
+ }
+ BuildMI(MBB, TI, DL,
+ TII->get(UseBKey ? AArch64::RETAB : AArch64::RETAA))
+ .copyImplicitOps(*MBBI)
+ .setMIFlag(MachineInstr::FrameDestroy);
+ }
+ MBB.erase(TI);
+ return;
+ }
+
+ for (auto *MI : SPMods)
+ MI->removeFromParent();
+
+ if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
+ assert(PACSym && "No PAC instruction to refer to");
+ BuildMI(MBB, MBBI, DL,
+ TII->get(UseBKey ? AArch64::AUTIBSPPCi : AArch64::AUTIASPPCi))
+ .addSym(PACSym)
+ .setMIFlag(MachineInstr::FrameDestroy);
+ emitAUTCFI(MBB, MBBI, EmitAsyncCFI);
+ } else {
+ if (MFnI->branchProtectionPAuthLR()) {
+ emitEpiloguePACSymOffsetIntoReg(*TII, MBB, MBBI, DL, PACSym,
+ AArch64::X16);
+
+ BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACM))
+ .setMIFlag(MachineInstr::FrameDestroy);
+ }
+ BuildMI(MBB, MBBI, DL,
+ TII->get(UseBKey ? AArch64::AUTIBSP : AArch64::AUTIASP))
+ .setMIFlag(MachineInstr::FrameDestroy);
+ emitAUTCFI(MBB, MBBI, EmitAsyncCFI);
+ }
+
+ if (NeedsWinCFI) {
+ assert(UseBKey &&
+ "Windows SEH PAC unwind info only supports B-key signing");
+ BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PACSignLR))
+ .setMIFlag(MachineInstr::FrameDestroy);
+ }
+
+ for (auto *MI : SPMods)
+ MBB.insert(MBBI, MI);
+
+ return;
+ }
+
for (auto *MI : SPMods)
MI->removeFromParent();
+ // Otherwise there is an offset to the incoming SP, and we can't use the aut
+ // variants that hard-code SP. Reconstruct entry SP in x16 and authenticate
+ // using AUTI[AB]1716 (x17=LR, x16=entry_SP).
+ emitFrameOffset(MBB, MBBI, DL, AArch64::X16, AArch64::SP,
+ StackOffset::getFixed(Offset), TII,
+ MachineInstr::FrameDestroy);
+
+ auto emitMOV = [&](Register Dst, Register Src) {
+ BuildMI(MBB, MBBI, DL, TII->get(AArch64::ORRXrs), Dst)
+ .addReg(AArch64::XZR)
+ .addReg(Src)
+ .addImm(0)
+ .setMIFlag(MachineInstr::FrameDestroy);
+ };
+
if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
+ emitMOV(AArch64::X17, AArch64::LR);
+
assert(PACSym && "No PAC instruction to refer to");
- BuildMI(MBB, MBBI, DL,
- TII->get(UseBKey ? AArch64::AUTIBSPPCi : AArch64::AUTIASPPCi))
- .addSym(PACSym)
+ emitEpiloguePACSymOffsetIntoReg(*TII, MBB, MBBI, DL, PACSym, AArch64::X15);
+
+ unsigned AutOpc = UseBKey ? AArch64::AUTIB171615 : AArch64::AUTIA171615;
+ BuildMI(MBB, MBBI, DL, TII->get(AutOpc))
+ .setMIFlag(MachineInstr::FrameDestroy);
+ emitAUTCFI(MBB, MBBI, EmitAsyncCFI);
+
+ emitMOV(AArch64::LR, AArch64::X17);
+ } else if (MFnI->branchProtectionPAuthLR()) {
+ emitMOV(AArch64::X17, AArch64::LR);
+
+ assert(PACSym && "No PAC instruction to refer to");
+ emitEpiloguePACSymOffsetIntoReg(*TII, MBB, MBBI, DL, PACSym, AArch64::X15);
+
+ // The PACM hint-space instruction modifies the following AUTI[AB]1716
+ // to optionally take x15 as an extra operand depending on the
+ // presence of +pauth-lr at runtime. On machines without +pauth-lr, it
+ // behaves as a nop, and the address of the PACI[AB]SP in x15 is
+ // ignored.
+ BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACM))
+ .setMIFlag(MachineInstr::FrameDestroy);
+
+ unsigned AutOpc = UseBKey ? AArch64::AUTIB1716 : AArch64::AUTIA1716;
+ BuildMI(MBB, MBBI, DL, TII->get(AutOpc))
+ .setMIFlag(MachineInstr::FrameDestroy);
+ emitAUTCFI(MBB, MBBI, EmitAsyncCFI);
+
+ emitMOV(AArch64::LR, AArch64::X17);
+ } else if (Subtarget->hasPAuth()) {
+ BuildMI(MBB, MBBI, DL, TII->get(UseBKey ? AArch64::AUTIB : AArch64::AUTIA),
+ AArch64::LR)
+ .addUse(AArch64::LR)
+ .addUse(AArch64::X16)
.setMIFlag(MachineInstr::FrameDestroy);
emitAUTCFI(MBB, MBBI, EmitAsyncCFI);
} else {
- if (MFnI->branchProtectionPAuthLR()) {
- emitEpiloguePACSymOffsetIntoReg(*TII, MBB, MBBI, DL, PACSym,
- AArch64::X16);
+ emitMOV(AArch64::X17, AArch64::LR);
- BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACM))
- .setMIFlag(MachineInstr::FrameDestroy);
- }
- BuildMI(MBB, MBBI, DL,
- TII->get(UseBKey ? AArch64::AUTIBSP : AArch64::AUTIASP))
+ unsigned AutOpc = UseBKey ? AArch64::AUTIB1716 : AArch64::AUTIA1716;
+ BuildMI(MBB, MBBI, DL, TII->get(AutOpc))
.setMIFlag(MachineInstr::FrameDestroy);
emitAUTCFI(MBB, MBBI, EmitAsyncCFI);
+
+ emitMOV(AArch64::LR, AArch64::X17);
}
if (NeedsWinCFI) {
diff --git a/llvm/test/CodeGen/AArch64/ptrauth-tail-call-stackadjust.ll b/llvm/test/CodeGen/AArch64/ptrauth-tail-call-stackadjust.ll
new file mode 100644
index 0000000000000..0db7d6e4264e0
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ptrauth-tail-call-stackadjust.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=arm64e-apple-macosx -o - | FileCheck %s
+
+; Test that epilogue LR authentication uses the correct entry-SP modifier
+; when local-frame teardown and callee-popped argument cleanup are combined
+; into a single SP adjustment: the entry SP is reconstructed into a scratch
+; register (x16) and authentication uses the register form of auti[ab]
+; rather than auti[ab]sp, deferring the real SP adjustment to after aut.
+
+%large_struct = type { i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64, i64 }
+
+declare void @external_func()
+
+define swifttailcc void @test_frame_and_args(%large_struct %s) #0 {
+; CHECK-LABEL: test_frame_and_args:
+; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: pacibsp
+; CHECK-NEXT: sub sp, sp, #80
+; CHECK-NEXT: str x30, [sp, #64] ; 8-byte Spill
+; CHECK-NEXT: .cfi_def_cfa_offset 80
+; CHECK-NEXT: .cfi_offset w30, -16
+; CHECK-NEXT: bl _external_func
+; CHECK-NEXT: ldr x30, [sp, #64] ; 8-byte Reload
+; CHECK-NEXT: add x16, sp, #80
+; CHECK-NEXT: autib x30, x16
+; CHECK-NEXT: add sp, sp, #144
+; CHECK-NEXT: ret
+entry:
+ %local1 = alloca [32 x i8], align 8
+ %local2 = alloca [32 x i8], align 8
+ call void @llvm.lifetime.start.p0(i64 32, ptr %local1)
+ call void @llvm.lifetime.start.p0(i64 32, ptr %local2)
+ call void @external_func()
+ call void @llvm.lifetime.end.p0(i64 32, ptr %local2)
+ call void @llvm.lifetime.end.p0(i64 32, ptr %local1)
+ ret void
+}
+
+declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+
+; Test the same combined teardown when the callee-popped argument space is
+; large enough that its cleanup is split into a shifted and unshifted parts.
+%large_struct2 = type { [600 x i64] }
+
+define swifttailcc void @test_frame_and_large_args(%large_struct2 %s) #0 {
+; CHECK-LABEL: test_frame_and_large_args:
+; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: pacibsp
+; CHECK-NEXT: sub sp, sp, #48
+; CHECK-NEXT: stp x28, x30, [sp, #32] ; 16-byte Folded Spill
+; CHECK-NEXT: .cfi_def_cfa_offset 48
+; CHECK-NEXT: .cfi_offset w30, -8
+; CHECK-NEXT: .cfi_offset w28, -16
+; CHECK-NEXT: bl _external_func
+; CHECK-NEXT: ldp x28, x30, [sp, #32] ; 16-byte Folded Reload
+; CHECK-NEXT: add x16, sp, #48
+; CHECK-NEXT: autib x30, x16
+; CHECK-NEXT: add sp, sp, #688
+; CHECK-NEXT: add sp, sp, #1, lsl #12 ; =4096
+; CHECK-NEXT: ret
+entry:
+ %local1 = alloca [32 x i8], align 8
+ call void @llvm.lifetime.start.p0(i64 32, ptr %local1)
+ call void @external_func()
+ call void @llvm.lifetime.end.p0(i64 32, ptr %local1)
+ ret void
+}
+
+attributes #0 = { "ptrauth-returns" "sign-return-address"="all" "sign-return-address-key"="b_key" }
More information about the llvm-commits
mailing list