[llvm] d093631 - [AArch64][PAC] Don't move sp adjustments to form better epilogue AUTs (#221009)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 17 13:53:47 PDT 2026
Author: Jon Roelofs
Date: 2026-09-17T13:53:39-07:00
New Revision: d093631e8ea2e7c9c0c7d866ba6cc62b742d6467
URL: https://github.com/llvm/llvm-project/commit/d093631e8ea2e7c9c0c7d866ba6cc62b742d6467
DIFF: https://github.com/llvm/llvm-project/commit/d093631e8ea2e7c9c0c7d866ba6cc62b742d6467.diff
LOG: [AArch64][PAC] Don't move sp adjustments to form better epilogue AUTs (#221009)
This has proven to be pretty fragile, and generally unsound. Instead, we
should treat the location of the PAUTH_EPILOGUE as _the_ location of the
AUT, and accept whatever that means in terms of the codegen we have to
emit for that pseudo at the time we emit it.
Added:
Modified:
llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
llvm/test/CodeGen/AArch64/aarch64-signedreturnaddress.ll
llvm/test/CodeGen/AArch64/ptrauth-tail-call-stackadjust.ll
llvm/test/CodeGen/AArch64/sign-return-address-pauthlr-slh.ll
llvm/test/CodeGen/AArch64/swifttail-ptrauth.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp b/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
index 0e4a51286cf54..ac1092ee52c8a 100644
--- a/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PointerAuth.cpp
@@ -309,81 +309,44 @@ void AArch64PointerAuthImpl::authenticateLR(
MF.getSubtarget().getFrameLowering());
int64_t ArgumentStackToRestore = AFL.getArgumentStackToRestore(MF, MBB);
- // When ArgumentStackToRestore > 0, this function received more argument
- // space than the tail callee pops. The epilogue contains an SP adjustment
- // (e.g. "add sp, sp, #N") to discard the leftover argument space. We must
- // authenticate *before* that adjustment so that AUTI[AB]SP sees the entry
- // SP discriminator. Move any such SP-adjusting instructions to after the
- // authentication instruction.
- //
- // When ArgumentStackToRestore < 0, the tail callee pops more argument space
- // than this function received, so after the frame teardown, SP is below the
- // entry SP used as the signing modifier.
- //
- // We cannot simply bump SP first and then use AUTI[AB]SP with the bumped
- // value, because the live arguments would fall below SP and potentially
- // outside the red-zone. Collect those SP adjustments in case we need to move
- // them after the AUT.
- int64_t Offset = -ArgumentStackToRestore;
- SmallVector<MachineInstr *, 2> SPMods;
- if (ArgumentStackToRestore > 0) {
- for (MachineInstr &MI : make_range(MBBI.getReverse(), MBB.rend())) {
- if (!MI.getFlag(MachineInstr::FrameDestroy))
- break;
- if ((MI.getOpcode() == AArch64::ADDXri ||
- MI.getOpcode() == AArch64::SUBXri) &&
- MI.getOperand(0).getReg() == AArch64::SP &&
- MI.getOperand(1).getReg() == AArch64::SP) {
- SPMods.push_back(&MI);
- int64_t Imm = MI.getOperand(2).getImm()
- << AArch64_AM::getShiftValue(MI.getOperand(3).getImm());
- Offset += MI.getOpcode() == AArch64::ADDXri ? Imm : -Imm;
- }
- }
- }
-
- // If there will not be an SP bump afterward, we can use an AUT or RET form
- // with a hardcoded SP discriminator.
- if (!Offset) {
- // The AUTIASP instruction assembles to a hint instruction before v8.3a so
- // this instruction can safely be used for any v8a architecture.
- // From v8.3a onwards there are optimised authenticate LR and return
- // instructions, namely RETA{A,B}, that can be used instead. In this case
- // the DW_CFA_AARCH64_negate_ra_state can't be emitted. Additionally,
- // RET{A,B} requires the SP to match its incoming value on entry to the
- // function.
- bool TerminatorIsCombinable = TI != MBB.end() &&
- TI->getOpcode() == AArch64::RET &&
- ArgumentStackToRestore == 0;
-
- if (Subtarget->hasPAuth() && TerminatorIsCombinable && !NeedsWinCFI &&
- !MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) {
- if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
- assert(PACSym && "No PAC instruction to refer to");
- BuildMI(MBB, TI, DL,
- TII->get(UseBKey ? AArch64::RETABSPPCi : AArch64::RETAASPPCi))
- .addSym(PACSym)
- .copyImplicitOps(*MBBI)
- .setMIFlag(MachineInstr::FrameDestroy);
- } else {
- if (MFnI->branchProtectionPAuthLR()) {
- emitEpiloguePACSymOffsetIntoReg(*TII, MBB, MBBI, DL, PACSym,
- AArch64::X16);
- BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACM))
- .setMIFlag(MachineInstr::FrameDestroy);
- }
- BuildMI(MBB, TI, DL,
- TII->get(UseBKey ? AArch64::RETAB : AArch64::RETAA))
- .copyImplicitOps(*MBBI)
+ // The AUTIASP instruction assembles to a hint instruction before v8.3a so
+ // this instruction can safely be used for any v8a architecture.
+ // From v8.3a onwards there are optimised authenticate LR and return
+ // instructions, namely RETA{A,B}, that can be used instead. In this case
+ // the DW_CFA_AARCH64_negate_ra_state can't be emitted. Additionally,
+ // RET{A,B} requires the SP to match its incoming value on entry to the
+ // function.
+ bool TerminatorIsCombinable = std::next(MBBI) == TI && TI != MBB.end() &&
+ TI->getOpcode() == AArch64::RET &&
+ ArgumentStackToRestore == 0;
+
+ if (Subtarget->hasPAuth() && TerminatorIsCombinable && !NeedsWinCFI &&
+ !MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) {
+ if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
+ assert(PACSym && "No PAC instruction to refer to");
+ BuildMI(MBB, TI, DL,
+ TII->get(UseBKey ? AArch64::RETABSPPCi : AArch64::RETAASPPCi))
+ .addSym(PACSym)
+ .copyImplicitOps(*MBBI)
+ .setMIFlag(MachineInstr::FrameDestroy);
+ } else {
+ if (MFnI->branchProtectionPAuthLR()) {
+ emitEpiloguePACSymOffsetIntoReg(*TII, MBB, MBBI, DL, PACSym,
+ AArch64::X16);
+ BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACM))
.setMIFlag(MachineInstr::FrameDestroy);
}
- MBB.erase(TI);
- return;
+ BuildMI(MBB, TI, DL, TII->get(UseBKey ? AArch64::RETAB : AArch64::RETAA))
+ .copyImplicitOps(*MBBI)
+ .setMIFlag(MachineInstr::FrameDestroy);
}
+ MBB.erase(TI);
+ return;
+ }
- for (auto *MI : SPMods)
- MI->removeFromParent();
-
+ // If PAUTH_EPILOGUE is at insertion point with a net zero offset on SP, we
+ // can use an AUT form with a hardcoded SP discriminator.
+ if (ArgumentStackToRestore == 0) {
if (MFnI->branchProtectionPAuthLR() && Subtarget->hasPAuthLR()) {
assert(PACSym && "No PAC instruction to refer to");
BuildMI(MBB, MBBI, DL,
@@ -412,20 +375,26 @@ void AArch64PointerAuthImpl::authenticateLR(
.setMIFlag(MachineInstr::FrameDestroy);
}
- for (auto *MI : SPMods)
- MBB.insert(MBBI, MI);
-
return;
}
- for (auto *MI : SPMods)
- MI->removeFromParent();
-
- // Otherwise there is an offset to the incoming SP, and we can't use the aut
- // variants that hard-code SP. Reconstruct entry SP in x16 and authenticate
- // using AUTI[AB]1716 (x17=LR, x16=entry_SP).
+ // When ArgumentStackToRestore > 0, this function received more argument
+ // space than the tail callee pops. The epilogue contains an SP adjustment
+ // (e.g. "add sp, sp, #N") to discard the leftover argument space.
+ //
+ // When ArgumentStackToRestore < 0, the tail callee pops more argument space
+ // than this function received, so after the frame teardown, SP is below the
+ // entry SP used as the signing modifier.
+ //
+ // We cannot simply bump SP first and then use AUTI[AB]SP with the bumped
+ // value, because the live arguments would fall below SP and potentially
+ // outside the red-zone.
+ //
+ // At this point there is an offset to the incoming SP, and we can't use the
+ // aut variants that hard-code SP. Reconstruct entry SP in x16 and
+ // authenticate using AUTI[AB]1716 (x17=LR, x16=entry_SP).
emitFrameOffset(MBB, MBBI, DL, AArch64::X16, AArch64::SP,
- StackOffset::getFixed(Offset), TII,
+ StackOffset::getFixed(-ArgumentStackToRestore), TII,
MachineInstr::FrameDestroy);
auto emitMOV = [&](Register Dst, Register Src) {
@@ -492,9 +461,6 @@ void AArch64PointerAuthImpl::authenticateLR(
BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PACSignLR))
.setMIFlag(MachineInstr::FrameDestroy);
}
-
- for (auto *MI : SPMods)
- MBB.insert(MBBI, MI);
}
unsigned llvm::AArch64PAuth::getCheckerSizeInBytes(AuthCheckMethod Method) {
diff --git a/llvm/test/CodeGen/AArch64/aarch64-signedreturnaddress.ll b/llvm/test/CodeGen/AArch64/aarch64-signedreturnaddress.ll
index 58db1923c6c66..6759c9c0684f5 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-signedreturnaddress.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-signedreturnaddress.ll
@@ -34,8 +34,9 @@ entry:
; CHECK-NEXT: ret
; CHECKV83: paciasp
; CHECKV83-NEXT: mov x0, x30
+; CHECKV83-NEXT: autiasp
; CHECKV83-NEXT: xpaci x0
-; CHECKV83-NEXT: retaa
+; CHECKV83-NEXT: ret
%0 = tail call ptr @llvm.returnaddress(i32 0)
ret ptr %0
}
diff --git a/llvm/test/CodeGen/AArch64/ptrauth-tail-call-stackadjust.ll b/llvm/test/CodeGen/AArch64/ptrauth-tail-call-stackadjust.ll
index 0db7d6e4264e0..fa1a77fea3cd9 100644
--- a/llvm/test/CodeGen/AArch64/ptrauth-tail-call-stackadjust.ll
+++ b/llvm/test/CodeGen/AArch64/ptrauth-tail-call-stackadjust.ll
@@ -21,9 +21,9 @@ define swifttailcc void @test_frame_and_args(%large_struct %s) #0 {
; CHECK-NEXT: .cfi_offset w30, -16
; CHECK-NEXT: bl _external_func
; CHECK-NEXT: ldr x30, [sp, #64] ; 8-byte Reload
-; CHECK-NEXT: add x16, sp, #80
-; CHECK-NEXT: autib x30, x16
; CHECK-NEXT: add sp, sp, #144
+; CHECK-NEXT: sub x16, sp, #64
+; CHECK-NEXT: autib x30, x16
; CHECK-NEXT: ret
entry:
%local1 = alloca [32 x i8], align 8
@@ -54,10 +54,11 @@ define swifttailcc void @test_frame_and_large_args(%large_struct2 %s) #0 {
; CHECK-NEXT: .cfi_offset w28, -16
; CHECK-NEXT: bl _external_func
; CHECK-NEXT: ldp x28, x30, [sp, #32] ; 16-byte Folded Reload
-; CHECK-NEXT: add x16, sp, #48
-; CHECK-NEXT: autib x30, x16
-; CHECK-NEXT: add sp, sp, #688
; CHECK-NEXT: add sp, sp, #1, lsl #12 ; =4096
+; CHECK-NEXT: add sp, sp, #688
+; CHECK-NEXT: sub x16, sp, #1, lsl #12 ; =4096
+; CHECK-NEXT: sub x16, x16, #640
+; CHECK-NEXT: autib x30, x16
; CHECK-NEXT: ret
entry:
%local1 = alloca [32 x i8], align 8
@@ -67,4 +68,44 @@ entry:
ret void
}
+declare swifttailcc void @callee_stack0()
+declare void @use(ptr)
+
+define swifttailcc void @test_frame_and_args_split_by_csr_reload([8 x i64], i64 %x) "sign-return-address"="all" "frame-pointer"="all" uwtable(async) {
+; CHECK-LABEL: test_frame_and_args_split_by_csr_reload:
+; CHECK: ; %bb.0: ; %entry
+; CHECK-NEXT: paciasp
+; CHECK-NEXT: str x28, [sp, #-32]! ; 8-byte Folded Spill
+; CHECK-NEXT: .cfi_def_cfa_offset 32
+; CHECK-NEXT: stp x29, x30, [sp, #16] ; 16-byte Folded Spill
+; CHECK-NEXT: add x29, sp, #16
+; CHECK-NEXT: .cfi_def_cfa w29, 16
+; CHECK-NEXT: .cfi_offset w30, -8
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: .cfi_offset w28, -32
+; CHECK-NEXT: sub sp, sp, #1, lsl #12 ; =4096
+; CHECK-NEXT: sub sp, sp, #3904
+; CHECK-NEXT: mov x0, sp
+; CHECK-NEXT: bl _use
+; CHECK-NEXT: add sp, sp, #1, lsl #12 ; =4096
+; CHECK-NEXT: add sp, sp, #3904
+; CHECK-NEXT: .cfi_def_cfa wsp, 32
+; CHECK-NEXT: ldp x29, x30, [sp, #16] ; 16-byte Folded Reload
+; CHECK-NEXT: ldr x28, [sp], #32 ; 8-byte Folded Reload
+; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: add sp, sp, #16
+; CHECK-NEXT: .cfi_def_cfa_offset -16
+; CHECK-NEXT: .cfi_restore w30
+; CHECK-NEXT: .cfi_restore w29
+; CHECK-NEXT: .cfi_restore w28
+; CHECK-NEXT: sub x16, sp, #16
+; CHECK-NEXT: autia x30, x16
+; CHECK-NEXT: b _callee_stack0
+entry:
+ %buf = alloca [8000 x i8], align 16
+ call void @use(ptr %buf)
+ tail call swifttailcc void @callee_stack0()
+ ret void
+}
+
attributes #0 = { "ptrauth-returns" "sign-return-address"="all" "sign-return-address-key"="b_key" }
diff --git a/llvm/test/CodeGen/AArch64/sign-return-address-pauthlr-slh.ll b/llvm/test/CodeGen/AArch64/sign-return-address-pauthlr-slh.ll
index fa3ebf3474291..6f3a1f8f90af5 100644
--- a/llvm/test/CodeGen/AArch64/sign-return-address-pauthlr-slh.ll
+++ b/llvm/test/CodeGen/AArch64/sign-return-address-pauthlr-slh.ll
@@ -89,9 +89,10 @@ define i32 @f() #0 {
; CHECK-PAUTH-PAUTHLR-NEXT: and x30, x30, x16
; CHECK-PAUTH-PAUTHLR-NEXT: csdb
; CHECK-PAUTH-PAUTHLR-NEXT: mov x1, sp
+; CHECK-PAUTH-PAUTHLR-NEXT: autiasppc .Ltmp0
; CHECK-PAUTH-PAUTHLR-NEXT: and x1, x1, x16
; CHECK-PAUTH-PAUTHLR-NEXT: mov sp, x1
-; CHECK-PAUTH-PAUTHLR-NEXT: retaasppc .Ltmp0
+; CHECK-PAUTH-PAUTHLR-NEXT: ret
entry:
%0 = tail call ptr @llvm.returnaddress(i32 0)
tail call void asm sideeffect "", "r"(ptr %0)
diff --git a/llvm/test/CodeGen/AArch64/swifttail-ptrauth.ll b/llvm/test/CodeGen/AArch64/swifttail-ptrauth.ll
index 81c8a214c3ba9..54191b3e4b6e7 100644
--- a/llvm/test/CodeGen/AArch64/swifttail-ptrauth.ll
+++ b/llvm/test/CodeGen/AArch64/swifttail-ptrauth.ll
@@ -119,29 +119,38 @@ define swifttailcc void @caller_to0_from8([8 x i64], i64) "branch-protection-pau
; CHECK-NEXT: .cfi_def_cfa wsp, 16
; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload
; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: add sp, sp, #16
; CHECK-NEXT: .cfi_def_cfa_offset -16
; CHECK-NEXT: .cfi_restore w30
; CHECK-NEXT: .cfi_restore w29
+; CHECK-NEXT: sub x16, sp, #16
-; COMPAT-NEXT: adrp x16, .Ltmp1
-; COMPAT-NEXT: add x16, x16, :lo12:.Ltmp1
+; COMPAT-NEXT: mov x17, x30
+; COMPAT-NEXT: adrp x15, .Ltmp1
+; COMPAT-NEXT: add x15, x15, :lo12:.Ltmp1
; COMPAT-NEXT: hint #39
-; COMPAT-NEXT: hint #29
+; COMPAT-NEXT: hint #12
; COMPAT-NEXT: .cfi_set_ra_state 0, 0
+; COMPAT-NEXT: mov x30, x17
-; V83A-NEXT: adrp x16, .Ltmp1
-; V83A-NEXT: add x16, x16, :lo12:.Ltmp1
+; V83A-NEXT: mov x17, x30
+; V83A-NEXT: adrp x15, .Ltmp1
+; V83A-NEXT: add x15, x15, :lo12:.Ltmp1
; V83A-NEXT: hint #39
-; V83A-NEXT: autiasp
+; V83A-NEXT: autia1716
; V83A-NEXT: .cfi_set_ra_state 0, 0
+; V83A-NEXT: mov x30, x17
-; V9A-NEXT: autiasppc .Ltmp1
+; V9A-NEXT: mov x17, x30
+; V9A-NEXT: adrp x15, .Ltmp1
+; V9A-NEXT: add x15, x15, :lo12:.Ltmp1
+; V9A-NEXT: autia171615
; V9A-NEXT: .cfi_set_ra_state 0, 0
+; V9A-NEXT: mov x30, x17
-; PAUTH-NEXT: autiasp
+; PAUTH-NEXT: autia x30, x16
; PAUTH-NEXT: .cfi_negate_ra_state
-; CHECK-NEXT: add sp, sp, #16
; CHECK-NEXT: b callee_stack0
; MIR-LABEL: name: caller_to0_from8
@@ -266,29 +275,38 @@ define swifttailcc void @crash_tc(i1 %c, [8 x i64] %pad, i64 %x) "branch-protect
; CHECK-NEXT: .cfi_def_cfa wsp, 16
; CHECK-NEXT: ldp x29, x30, [sp], #16
; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: add sp, sp, #80
; CHECK-NEXT: .cfi_def_cfa_offset -80
; CHECK-NEXT: .cfi_restore w30
; CHECK-NEXT: .cfi_restore w29
+; CHECK-NEXT: sub x16, sp, #80
-; COMPAT-NEXT: adrp x16, .Ltmp3
-; COMPAT-NEXT: add x16, x16, :lo12:.Ltmp3
+; COMPAT-NEXT: mov x17, x30
+; COMPAT-NEXT: adrp x15, .Ltmp3
+; COMPAT-NEXT: add x15, x15, :lo12:.Ltmp3
; COMPAT-NEXT: hint #39
-; COMPAT-NEXT: hint #29
+; COMPAT-NEXT: hint #12
; COMPAT-NEXT: .cfi_set_ra_state 0, 0
+; COMPAT-NEXT: mov x30, x17
-; V83A-NEXT: adrp x16, .Ltmp3
-; V83A-NEXT: add x16, x16, :lo12:.Ltmp3
+; V83A-NEXT: mov x17, x30
+; V83A-NEXT: adrp x15, .Ltmp3
+; V83A-NEXT: add x15, x15, :lo12:.Ltmp3
; V83A-NEXT: hint #39
-; V83A-NEXT: autiasp
+; V83A-NEXT: autia1716
; V83A-NEXT: .cfi_set_ra_state 0, 0
+; V83A-NEXT: mov x30, x17
-; V9A-NEXT: autiasppc .Ltmp3
+; V9A-NEXT: mov x17, x30
+; V9A-NEXT: adrp x15, .Ltmp3
+; V9A-NEXT: add x15, x15, :lo12:.Ltmp3
+; V9A-NEXT: autia171615
; V9A-NEXT: .cfi_set_ra_state 0, 0
+; V9A-NEXT: mov x30, x17
-; PAUTH-NEXT: autiasp
+; PAUTH-NEXT: autia x30, x16
; PAUTH-NEXT: .cfi_negate_ra_state
-; CHECK-NEXT: add sp, sp, #80
; CHECK-NEXT: ret
; CHECK-NEXT: .LBB3_2:
; CHECK-NEXT: .cfi_restore_state
@@ -296,29 +314,38 @@ define swifttailcc void @crash_tc(i1 %c, [8 x i64] %pad, i64 %x) "branch-protect
; CHECK-NEXT: .cfi_def_cfa wsp, 16
; CHECK-NEXT: ldp x29, x30, [sp], #16
; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: add sp, sp, #80
; CHECK-NEXT: .cfi_def_cfa_offset -80
; CHECK-NEXT: .cfi_restore w30
; CHECK-NEXT: .cfi_restore w29
+; CHECK-NEXT: sub x16, sp, #80
-; COMPAT-NEXT: adrp x16, .Ltmp3
-; COMPAT-NEXT: add x16, x16, :lo12:.Ltmp3
+; COMPAT-NEXT: mov x17, x30
+; COMPAT-NEXT: adrp x15, .Ltmp3
+; COMPAT-NEXT: add x15, x15, :lo12:.Ltmp3
; COMPAT-NEXT: hint #39
-; COMPAT-NEXT: hint #29
+; COMPAT-NEXT: hint #12
; COMPAT-NEXT: .cfi_set_ra_state 0, 0
+; COMPAT-NEXT: mov x30, x17
-; V83A-NEXT: adrp x16, .Ltmp3
-; V83A-NEXT: add x16, x16, :lo12:.Ltmp3
+; V83A-NEXT: mov x17, x30
+; V83A-NEXT: adrp x15, .Ltmp3
+; V83A-NEXT: add x15, x15, :lo12:.Ltmp3
; V83A-NEXT: hint #39
-; V83A-NEXT: autiasp
+; V83A-NEXT: autia1716
; V83A-NEXT: .cfi_set_ra_state 0, 0
+; V83A-NEXT: mov x30, x17
-; V9A-NEXT: autiasppc .Ltmp3
+; V9A-NEXT: mov x17, x30
+; V9A-NEXT: adrp x15, .Ltmp3
+; V9A-NEXT: add x15, x15, :lo12:.Ltmp3
+; V9A-NEXT: autia171615
; V9A-NEXT: .cfi_set_ra_state 0, 0
+; V9A-NEXT: mov x30, x17
-; PAUTH-NEXT: autiasp
+; PAUTH-NEXT: autia x30, x16
; PAUTH-NEXT: .cfi_negate_ra_state
-; CHECK-NEXT: add sp, sp, #80
; CHECK-NEXT: b callee_stack0
; MIR-LABEL: name: crash_tc
More information about the llvm-commits
mailing list