[llvm] [AArch64] Break up `AArch64FrameLowering::emitEpilogue` (NFCI) (PR #157889)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 10 09:08:50 PDT 2025
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Benjamin Maxwell (MacDue)
<details>
<summary>Changes</summary>
This is much smaller than #<!-- -->157485 (as the epilogue code was already a ore reasonable size); however, this change will allow some further tidy up of methods shared between the prologue and epilogue code (in a follow-up patch).
---
Patch is 45.61 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/157889.diff
4 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64FrameLowering.cpp (+5-449)
- (modified) llvm/lib/Target/AArch64/AArch64FrameLowering.h (+13-5)
- (modified) llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp (+432)
- (modified) llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h (+60-2)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 175b5e04d82ff..fd53f04443766 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -293,14 +293,9 @@ static cl::opt<bool> DisableMultiVectorSpillFill(
cl::desc("Disable use of LD/ST pairs for SME2 or SVE2p1"), cl::init(false),
cl::Hidden);
-/// Returns how much of the incoming argument stack area (in bytes) we should
-/// clean up in an epilogue. For the C calling convention this will be 0, for
-/// guaranteed tail call conventions it can be positive (a normal return or a
-/// tail call to a function that uses less stack space for arguments) or
-/// negative (for a tail call to a function that needs more stack space than us
-/// for arguments).
-static int64_t getArgumentStackToRestore(MachineFunction &MF,
- MachineBasicBlock &MBB) {
+int64_t
+AArch64FrameLowering::getArgumentStackToRestore(MachineFunction &MF,
+ MachineBasicBlock &MBB) const {
MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
bool IsTailCallReturn = (MBB.end() != MBBI)
@@ -711,44 +706,6 @@ void AArch64FrameLowering::resetCFIToInitialState(
}
}
-static void emitCalleeSavedRestores(MachineBasicBlock &MBB,
- MachineBasicBlock::iterator MBBI,
- bool SVE) {
- MachineFunction &MF = *MBB.getParent();
- MachineFrameInfo &MFI = MF.getFrameInfo();
-
- const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
- if (CSI.empty())
- return;
-
- const TargetSubtargetInfo &STI = MF.getSubtarget();
- const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
- CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameDestroy);
-
- for (const auto &Info : CSI) {
- if (SVE !=
- (MFI.getStackID(Info.getFrameIdx()) == TargetStackID::ScalableVector))
- continue;
-
- MCRegister Reg = Info.getReg();
- if (SVE &&
- !static_cast<const AArch64RegisterInfo &>(TRI).regNeedsCFI(Reg, Reg))
- continue;
-
- CFIBuilder.buildRestore(Info.getReg());
- }
-}
-
-void AArch64FrameLowering::emitCalleeSavedGPRRestores(
- MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
- emitCalleeSavedRestores(MBB, MBBI, false);
-}
-
-void AArch64FrameLowering::emitCalleeSavedSVERestores(
- MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
- emitCalleeSavedRestores(MBB, MBBI, true);
-}
-
// Return the maximum possible number of bytes for `Size` due to the
// architectural limit on the size of a SVE register.
static int64_t upperBound(StackOffset Size) {
@@ -1642,28 +1599,6 @@ bool AArch64FrameLowering::isSVECalleeSave(
}
}
-static void emitShadowCallStackEpilogue(const TargetInstrInfo &TII,
- MachineFunction &MF,
- MachineBasicBlock &MBB,
- MachineBasicBlock::iterator MBBI,
- const DebugLoc &DL, bool NeedsWinCFI) {
- // Shadow call stack epilog: ldr x30, [x18, #-8]!
- BuildMI(MBB, MBBI, DL, TII.get(AArch64::LDRXpre))
- .addReg(AArch64::X18, RegState::Define)
- .addReg(AArch64::LR, RegState::Define)
- .addReg(AArch64::X18)
- .addImm(-8)
- .setMIFlag(MachineInstr::FrameDestroy);
-
- if (NeedsWinCFI)
- BuildMI(MBB, MBBI, DL, TII.get(AArch64::SEH_Nop))
- .setMIFlag(MachineInstr::FrameDestroy);
-
- if (MF.getInfo<AArch64FunctionInfo>()->needsAsyncDwarfUnwindInfo(MF))
- CFIInstBuilder(MBB, MBBI, MachineInstr::FrameDestroy)
- .buildRestore(AArch64::X18);
-}
-
void AArch64FrameLowering::emitPacRetPlusLeafHardening(
MachineFunction &MF) const {
const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
@@ -1703,389 +1638,10 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
PrologueEmitter.emitPrologue();
}
-static bool isFuncletReturnInstr(const MachineInstr &MI) {
- switch (MI.getOpcode()) {
- default:
- return false;
- case AArch64::CATCHRET:
- case AArch64::CLEANUPRET:
- return true;
- }
-}
-
void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
MachineBasicBlock &MBB) const {
- MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
- MachineFrameInfo &MFI = MF.getFrameInfo();
- AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
- const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
- const TargetInstrInfo *TII = Subtarget.getInstrInfo();
- DebugLoc DL;
- bool NeedsWinCFI = needsWinCFI(MF);
- bool EmitCFI = AFI->needsAsyncDwarfUnwindInfo(MF);
- bool HasWinCFI = false;
- bool IsFunclet = false;
-
- if (MBB.end() != MBBI) {
- DL = MBBI->getDebugLoc();
- IsFunclet = isFuncletReturnInstr(*MBBI);
- }
-
- MachineBasicBlock::iterator EpilogStartI = MBB.end();
-
- auto FinishingTouches = make_scope_exit([&]() {
- if (AFI->needsShadowCallStackPrologueEpilogue(MF)) {
- emitShadowCallStackEpilogue(*TII, MF, MBB, MBB.getFirstTerminator(), DL,
- NeedsWinCFI);
- HasWinCFI |= NeedsWinCFI;
- }
- if (EmitCFI)
- emitCalleeSavedGPRRestores(MBB, MBB.getFirstTerminator());
- if (AFI->shouldSignReturnAddress(MF)) {
- // If pac-ret+leaf is in effect, PAUTH_EPILOGUE pseudo instructions
- // are inserted by emitPacRetPlusLeafHardening().
- if (!shouldSignReturnAddressEverywhere(MF)) {
- BuildMI(MBB, MBB.getFirstTerminator(), DL,
- TII->get(AArch64::PAUTH_EPILOGUE))
- .setMIFlag(MachineInstr::FrameDestroy);
- }
- // AArch64PointerAuth pass will insert SEH_PACSignLR
- HasWinCFI |= NeedsWinCFI;
- }
- if (HasWinCFI) {
- BuildMI(MBB, MBB.getFirstTerminator(), DL,
- TII->get(AArch64::SEH_EpilogEnd))
- .setMIFlag(MachineInstr::FrameDestroy);
- if (!MF.hasWinCFI())
- MF.setHasWinCFI(true);
- }
- if (NeedsWinCFI) {
- assert(EpilogStartI != MBB.end());
- if (!HasWinCFI)
- MBB.erase(EpilogStartI);
- }
- });
-
- int64_t NumBytes = IsFunclet ? getWinEHFuncletFrameSize(MF)
- : MFI.getStackSize();
-
- // All calls are tail calls in GHC calling conv, and functions have no
- // prologue/epilogue.
- if (MF.getFunction().getCallingConv() == CallingConv::GHC)
- return;
-
- // How much of the stack used by incoming arguments this function is expected
- // to restore in this particular epilogue.
- int64_t ArgumentStackToRestore = getArgumentStackToRestore(MF, MBB);
- bool IsWin64 = Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv(),
- MF.getFunction().isVarArg());
- unsigned FixedObject = getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
-
- int64_t AfterCSRPopSize = ArgumentStackToRestore;
- auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
- // We cannot rely on the local stack size set in emitPrologue if the function
- // has funclets, as funclets have different local stack size requirements, and
- // the current value set in emitPrologue may be that of the containing
- // function.
- if (MF.hasEHFunclets())
- AFI->setLocalStackSize(NumBytes - PrologueSaveSize);
- if (homogeneousPrologEpilog(MF, &MBB)) {
- assert(!NeedsWinCFI);
- auto FirstHomogenousEpilogI = MBB.getFirstTerminator();
- if (FirstHomogenousEpilogI != MBB.begin()) {
- auto HomogeneousEpilog = std::prev(FirstHomogenousEpilogI);
- if (HomogeneousEpilog->getOpcode() == AArch64::HOM_Epilog)
- FirstHomogenousEpilogI = HomogeneousEpilog;
- }
-
- // Adjust local stack
- emitFrameOffset(MBB, FirstHomogenousEpilogI, DL, AArch64::SP, AArch64::SP,
- StackOffset::getFixed(AFI->getLocalStackSize()), TII,
- MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
-
- // SP has been already adjusted while restoring callee save regs.
- // We've bailed-out the case with adjusting SP for arguments.
- assert(AfterCSRPopSize == 0);
- return;
- }
-
- bool FPAfterSVECalleeSaves =
- Subtarget.isTargetWindows() && AFI->getSVECalleeSavedStackSize();
-
- bool CombineSPBump = shouldCombineCSRLocalStackBumpInEpilogue(MBB, NumBytes);
- // Assume we can't combine the last pop with the sp restore.
- bool CombineAfterCSRBump = false;
- if (FPAfterSVECalleeSaves) {
- AfterCSRPopSize += FixedObject;
- } else if (!CombineSPBump && PrologueSaveSize != 0) {
- MachineBasicBlock::iterator Pop = std::prev(MBB.getFirstTerminator());
- while (Pop->getOpcode() == TargetOpcode::CFI_INSTRUCTION ||
- AArch64InstrInfo::isSEHInstruction(*Pop))
- Pop = std::prev(Pop);
- // Converting the last ldp to a post-index ldp is valid only if the last
- // ldp's offset is 0.
- const MachineOperand &OffsetOp = Pop->getOperand(Pop->getNumOperands() - 1);
- // If the offset is 0 and the AfterCSR pop is not actually trying to
- // allocate more stack for arguments (in space that an untimely interrupt
- // may clobber), convert it to a post-index ldp.
- if (OffsetOp.getImm() == 0 && AfterCSRPopSize >= 0) {
- convertCalleeSaveRestoreToSPPrePostIncDec(
- MBB, Pop, DL, TII, PrologueSaveSize, NeedsWinCFI, &HasWinCFI, EmitCFI,
- MachineInstr::FrameDestroy, PrologueSaveSize);
- } else {
- // If not, make sure to emit an add after the last ldp.
- // We're doing this by transferring the size to be restored from the
- // adjustment *before* the CSR pops to the adjustment *after* the CSR
- // pops.
- AfterCSRPopSize += PrologueSaveSize;
- CombineAfterCSRBump = true;
- }
- }
-
- // Move past the restores of the callee-saved registers.
- // If we plan on combining the sp bump of the local stack size and the callee
- // save stack size, we might need to adjust the CSR save and restore offsets.
- MachineBasicBlock::iterator FirstGPRRestoreI = MBB.getFirstTerminator();
- MachineBasicBlock::iterator Begin = MBB.begin();
- while (FirstGPRRestoreI != Begin) {
- --FirstGPRRestoreI;
- if (!FirstGPRRestoreI->getFlag(MachineInstr::FrameDestroy) ||
- (!FPAfterSVECalleeSaves && isSVECalleeSave(FirstGPRRestoreI))) {
- ++FirstGPRRestoreI;
- break;
- } else if (CombineSPBump)
- fixupCalleeSaveRestoreStackOffset(
- *FirstGPRRestoreI, AFI->getLocalStackSize(), NeedsWinCFI, &HasWinCFI);
- }
-
- if (NeedsWinCFI) {
- // Note that there are cases where we insert SEH opcodes in the
- // epilogue when we had no SEH opcodes in the prologue. For
- // example, when there is no stack frame but there are stack
- // arguments. Insert the SEH_EpilogStart and remove it later if it
- // we didn't emit any SEH opcodes to avoid generating WinCFI for
- // functions that don't need it.
- BuildMI(MBB, FirstGPRRestoreI, DL, TII->get(AArch64::SEH_EpilogStart))
- .setMIFlag(MachineInstr::FrameDestroy);
- EpilogStartI = FirstGPRRestoreI;
- --EpilogStartI;
- }
-
- if (hasFP(MF) && AFI->hasSwiftAsyncContext()) {
- switch (MF.getTarget().Options.SwiftAsyncFramePointer) {
- case SwiftAsyncFramePointerMode::DeploymentBased:
- // Avoid the reload as it is GOT relative, and instead fall back to the
- // hardcoded value below. This allows a mismatch between the OS and
- // application without immediately terminating on the difference.
- [[fallthrough]];
- case SwiftAsyncFramePointerMode::Always:
- // We need to reset FP to its untagged state on return. Bit 60 is
- // currently used to show the presence of an extended frame.
-
- // BIC x29, x29, #0x1000_0000_0000_0000
- BuildMI(MBB, MBB.getFirstTerminator(), DL, TII->get(AArch64::ANDXri),
- AArch64::FP)
- .addUse(AArch64::FP)
- .addImm(0x10fe)
- .setMIFlag(MachineInstr::FrameDestroy);
- if (NeedsWinCFI) {
- BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_Nop))
- .setMIFlags(MachineInstr::FrameDestroy);
- HasWinCFI = true;
- }
- break;
-
- case SwiftAsyncFramePointerMode::Never:
- break;
- }
- }
-
- const StackOffset &SVEStackSize = getSVEStackSize(MF);
-
- // If there is a single SP update, insert it before the ret and we're done.
- if (CombineSPBump) {
- assert(!SVEStackSize && "Cannot combine SP bump with SVE");
-
- // When we are about to restore the CSRs, the CFA register is SP again.
- if (EmitCFI && hasFP(MF))
- CFIInstBuilder(MBB, FirstGPRRestoreI, MachineInstr::FrameDestroy)
- .buildDefCFA(AArch64::SP, NumBytes);
-
- emitFrameOffset(MBB, MBB.getFirstTerminator(), DL, AArch64::SP, AArch64::SP,
- StackOffset::getFixed(NumBytes + AfterCSRPopSize), TII,
- MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI,
- EmitCFI, StackOffset::getFixed(NumBytes));
- return;
- }
-
- NumBytes -= PrologueSaveSize;
- assert(NumBytes >= 0 && "Negative stack allocation size!?");
-
- // Process the SVE callee-saves to determine what space needs to be
- // deallocated.
- StackOffset DeallocateBefore = {}, DeallocateAfter = SVEStackSize;
- MachineBasicBlock::iterator RestoreBegin = FirstGPRRestoreI,
- RestoreEnd = FirstGPRRestoreI;
- if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
- if (FPAfterSVECalleeSaves)
- RestoreEnd = MBB.getFirstTerminator();
-
- RestoreBegin = std::prev(RestoreEnd);
- while (RestoreBegin != MBB.begin() &&
- isSVECalleeSave(std::prev(RestoreBegin)))
- --RestoreBegin;
-
- assert(isSVECalleeSave(RestoreBegin) &&
- isSVECalleeSave(std::prev(RestoreEnd)) && "Unexpected instruction");
-
- StackOffset CalleeSavedSizeAsOffset =
- StackOffset::getScalable(CalleeSavedSize);
- DeallocateBefore = SVEStackSize - CalleeSavedSizeAsOffset;
- DeallocateAfter = CalleeSavedSizeAsOffset;
- }
-
- // Deallocate the SVE area.
- if (FPAfterSVECalleeSaves) {
- // If the callee-save area is before FP, restoring the FP implicitly
- // deallocates non-callee-save SVE allocations. Otherwise, deallocate
- // them explicitly.
- if (!AFI->isStackRealigned() && !MFI.hasVarSizedObjects()) {
- emitFrameOffset(MBB, FirstGPRRestoreI, DL, AArch64::SP, AArch64::SP,
- DeallocateBefore, TII, MachineInstr::FrameDestroy, false,
- NeedsWinCFI, &HasWinCFI);
- }
-
- // Deallocate callee-save non-SVE registers.
- emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
- StackOffset::getFixed(AFI->getCalleeSavedStackSize()), TII,
- MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
-
- // Deallocate fixed objects.
- emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
- StackOffset::getFixed(FixedObject), TII,
- MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
-
- // Deallocate callee-save SVE registers.
- emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
- DeallocateAfter, TII, MachineInstr::FrameDestroy, false,
- NeedsWinCFI, &HasWinCFI);
- } else if (SVEStackSize) {
- int64_t SVECalleeSavedSize = AFI->getSVECalleeSavedStackSize();
- // If we have stack realignment or variable-sized objects we must use the
- // FP to restore SVE callee saves (as there is an unknown amount of
- // data/padding between the SP and SVE CS area).
- Register BaseForSVEDealloc =
- (AFI->isStackRealigned() || MFI.hasVarSizedObjects()) ? AArch64::FP
- : AArch64::SP;
- if (SVECalleeSavedSize && BaseForSVEDealloc == AArch64::FP) {
- Register CalleeSaveBase = AArch64::FP;
- if (int64_t CalleeSaveBaseOffset =
- AFI->getCalleeSaveBaseToFrameRecordOffset()) {
- // If we have have an non-zero offset to the non-SVE CS base we need to
- // compute the base address by subtracting the offest in a temporary
- // register first (to avoid briefly deallocating the SVE CS).
- CalleeSaveBase = MBB.getParent()->getRegInfo().createVirtualRegister(
- &AArch64::GPR64RegClass);
- emitFrameOffset(MBB, RestoreBegin, DL, CalleeSaveBase, AArch64::FP,
- StackOffset::getFixed(-CalleeSaveBaseOffset), TII,
- MachineInstr::FrameDestroy);
- }
- // The code below will deallocate the stack space space by moving the
- // SP to the start of the SVE callee-save area.
- emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, CalleeSaveBase,
- StackOffset::getScalable(-SVECalleeSavedSize), TII,
- MachineInstr::FrameDestroy);
- } else if (BaseForSVEDealloc == AArch64::SP) {
- if (SVECalleeSavedSize) {
- // Deallocate the non-SVE locals first before we can deallocate (and
- // restore callee saves) from the SVE area.
- emitFrameOffset(
- MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
- StackOffset::getFixed(NumBytes), TII, MachineInstr::FrameDestroy,
- false, NeedsWinCFI, &HasWinCFI, EmitCFI && !hasFP(MF),
- SVEStackSize + StackOffset::getFixed(NumBytes + PrologueSaveSize));
- NumBytes = 0;
- }
-
- emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
- DeallocateBefore, TII, MachineInstr::FrameDestroy, false,
- NeedsWinCFI, &HasWinCFI, EmitCFI && !hasFP(MF),
- SVEStackSize +
- StackOffset::getFixed(NumBytes + PrologueSaveSize));
-
- emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
- DeallocateAfter, TII, MachineInstr::FrameDestroy, false,
- NeedsWinCFI, &HasWinCFI, EmitCFI && !hasFP(MF),
- DeallocateAfter +
- StackOffset::getFixed(NumBytes + PrologueSaveSize));
- }
- if (EmitCFI)
- emitCalleeSavedSVERestores(MBB, RestoreEnd);
- }
-
- if (!hasFP(MF)) {
- bool RedZone = canUseRedZone(MF);
- // If this was a redzone leaf function, we don't need to restore the
- // stack pointer (but we may need to pop stack args for fastcc).
- if (RedZone && AfterCSRPopSize == 0)
- return;
-
- // Pop the local variables off the stack. If there are no callee-saved
- // registers, it means we are actually positioned at the terminator and can
- // combine stack increment for the locals and the stack increment for
- // callee-popped arguments into (possibly) a single instruction and be done.
- bool NoCalleeSaveRestore = PrologueSaveSize == 0;
- int64_t StackRestoreBytes = RedZone ? 0 : NumBytes;
- if (NoCalleeSaveRestore)
- StackRestoreBytes += AfterCSRPopSize;
-
- emitFrameOffset(
- MBB, FirstGPRRestoreI, DL, AArch64::SP, AArch64::SP,
- StackOffset::getFixed(StackRestoreBytes), TII,
- MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI, EmitCFI,
- StackOffset::getFixed((RedZone ? 0 : NumBytes) + PrologueSaveSize));
-
- // If we were able to combine the local stack pop with the argument pop,
- // then we're done.
- if (NoCalleeSaveRestore || AfterCSRPopSize == 0) {
- return;
- }
-
- NumBytes = 0;
- }
-
- // Restore the original stack pointer.
- // FIXME: Rather than doing the math here, we should instead just use
- // non-post-indexed loads for the restores if we aren't actually going to
- // be able to save any instructions.
- if (!IsFunclet && (MFI.hasVarSizedObjects() || AFI->isStackRealigned())) {
- emitFrameOffset(
- MBB, FirstGPRRestoreI, DL, AArch64::SP, AArch64::FP,
- StackOffset::getFixed(-AFI->getCalleeSaveBaseToFrameRecordOffset()),
- TII, MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
- } else if (NumBytes)
- emitFrameOffset(MBB, FirstGPRRestoreI, DL, ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/157889
More information about the llvm-commits
mailing list