[llvm] [AArch64][SVE] Share code across SVE prologue/epilogue implementations (NFCI) (PR #162253)
Benjamin Maxwell via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 7 08:52:24 PDT 2025
https://github.com/MacDue updated https://github.com/llvm/llvm-project/pull/162253
>From f71c702872400aa2bf63fa4a358639170ddfb23e Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 6 Oct 2025 10:04:23 +0000
Subject: [PATCH 01/13] Step 1
---
.../AArch64/AArch64PrologueEpilogue.cpp | 144 +++++++++---------
.../Target/AArch64/AArch64PrologueEpilogue.h | 7 +
2 files changed, 81 insertions(+), 70 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
index 15681617f5c05..6c24fca272822 100644
--- a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
@@ -94,6 +94,26 @@ AArch64PrologueEpilogueCommon::AArch64PrologueEpilogueCommon(
HasFP = AFL.hasFP(MF);
NeedsWinCFI = AFL.needsWinCFI(MF);
+
+ // Windows unwind can't represent the required stack adjustments if we have
+ // both SVE callee-saves and dynamic stack allocations, and the frame
+ // pointer is before the SVE spills. The allocation of the frame pointer
+ // must be the last instruction in the prologue so the unwinder can restore
+ // the stack pointer correctly. (And there isn't any unwind opcode for
+ // `addvl sp, x29, -17`.)
+ //
+ // Because of this, we do spills in the opposite order on Windows: first SVE,
+ // then GPRs. The main side-effect of this is that it makes accessing
+ // parameters passed on the stack more expensive.
+ //
+ // We could consider rearranging the spills for simpler cases.
+ if (Subtarget.isTargetWindows() && AFI->getSVECalleeSavedStackSize()) {
+ if (AFI->hasStackHazardSlotIndex())
+ reportFatalUsageError("SME hazard padding is not supported on Windows");
+ SVELayout = SVEStackLayout::CalleeSavesAboveFrameRecord;
+ } else if (AFI->hasSplitSVEObjects()) {
+ SVELayout = SVEStackLayout::Split;
+ }
}
MachineBasicBlock::iterator
@@ -613,30 +633,12 @@ void AArch64PrologueEmitter::emitPrologue() {
bool IsWin64 = Subtarget.isCallingConvWin64(F.getCallingConv(), F.isVarArg());
unsigned FixedObject = AFL.getFixedObjectSize(MF, AFI, IsWin64, IsFunclet);
- // Windows unwind can't represent the required stack adjustments if we have
- // both SVE callee-saves and dynamic stack allocations, and the frame
- // pointer is before the SVE spills. The allocation of the frame pointer
- // must be the last instruction in the prologue so the unwinder can restore
- // the stack pointer correctly. (And there isn't any unwind opcode for
- // `addvl sp, x29, -17`.)
- //
- // Because of this, we do spills in the opposite order on Windows: first SVE,
- // then GPRs. The main side-effect of this is that it makes accessing
- // parameters passed on the stack more expensive.
- //
- // We could consider rearranging the spills for simpler cases.
- bool FPAfterSVECalleeSaves =
- Subtarget.isTargetWindows() && AFI->getSVECalleeSavedStackSize();
-
- if (FPAfterSVECalleeSaves && AFI->hasStackHazardSlotIndex())
- reportFatalUsageError("SME hazard padding is not supported on Windows");
-
auto PrologueSaveSize = AFI->getCalleeSavedStackSize() + FixedObject;
// All of the remaining stack allocations are for locals.
determineLocalsStackSize(NumBytes, PrologueSaveSize);
MachineBasicBlock::iterator FirstGPRSaveI = PrologueBeginI;
- if (FPAfterSVECalleeSaves) {
+ if (SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord) {
// If we're doing SVE saves first, we need to immediately allocate space
// for fixed objects, then space for the SVE callee saves.
//
@@ -726,7 +728,7 @@ void AArch64PrologueEmitter::emitPrologue() {
StackOffset CFAOffset =
StackOffset::getFixed((int64_t)MFI.getStackSize() - NumBytes);
MachineBasicBlock::iterator AfterSVESavesI = AfterGPRSavesI;
- if (!FPAfterSVECalleeSaves) {
+ if (SVELayout != SVEStackLayout::CalleeSavesAboveFrameRecord) {
// Process the SVE callee-saves to find the starts/ends of the ZPR and PPR
// areas.
PPRCalleeSavesBegin = AfterGPRSavesI;
@@ -756,9 +758,7 @@ void AArch64PrologueEmitter::emitPrologue() {
if (EmitAsyncCFI)
emitCalleeSavedSVELocations(AfterSVESavesI);
- if (AFI->hasSplitSVEObjects()) {
- assert(!FPAfterSVECalleeSaves &&
- "Cannot use FPAfterSVECalleeSaves with aarch64-split-sve-objects");
+ if (SVELayout == SVEStackLayout::Split) {
assert(!AFL.canUseRedZone(MF) &&
"Cannot use redzone with aarch64-split-sve-objects");
// TODO: Handle HasWinCFI/NeedsWinCFI?
@@ -791,7 +791,7 @@ void AArch64PrologueEmitter::emitPrologue() {
// Allocate space for the callee saves (if any).
StackOffset LocalsSize =
PPRLocalsSize + ZPRLocalsSize + StackOffset::getFixed(NumBytes);
- if (!FPAfterSVECalleeSaves)
+ if (SVELayout != SVEStackLayout::CalleeSavesAboveFrameRecord)
allocateStackSpace(AfterGPRSavesI, 0, SVECalleeSavesSize,
EmitAsyncCFI && !HasFP, CFAOffset,
MFI.hasVarSizedObjects() || LocalsSize);
@@ -1270,7 +1270,7 @@ void AArch64PrologueEmitter::emitCalleeSavedSVELocations(
StackOffset::getScalable(MFI.getObjectOffset(FI)) -
StackOffset::getFixed(AFI->getCalleeSavedStackSize(MFI));
- if (AFI->hasSplitSVEObjects() &&
+ if (SVELayout == SVEStackLayout::Split &&
MFI.getStackID(FI) == TargetStackID::ScalableVector)
Offset -= PPRStackSize;
@@ -1298,6 +1298,28 @@ AArch64EpilogueEmitter::AArch64EpilogueEmitter(MachineFunction &MF,
SEHEpilogueStartI = MBB.end();
}
+struct SVEEpiloguePartitions {
+ MachineBasicBlock::iterator RestoreBegin, RestoreEnd;
+};
+
+static SVEEpiloguePartitions
+partitionSVEEpilogue(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+ StackOffset SVECalleeSavesSize,
+ StackOffset SVELocalsSize) {
+ MachineBasicBlock::iterator RestoreBegin = MBBI, RestoreEnd = MBBI;
+ if (SVECalleeSavesSize) {
+ RestoreBegin = std::prev(RestoreEnd);
+ while (RestoreBegin != MBB.begin() &&
+ isPartOfSVECalleeSaves(std::prev(RestoreBegin)))
+ --RestoreBegin;
+
+ assert(isPartOfSVECalleeSaves(RestoreBegin) &&
+ isPartOfSVECalleeSaves(std::prev(RestoreEnd)) &&
+ "Unexpected instruction");
+ }
+ return {RestoreBegin, RestoreEnd};
+}
+
void AArch64EpilogueEmitter::emitEpilogue() {
MachineBasicBlock::iterator EpilogueEndI = MBB.getLastNonDebugInstr();
if (MBB.end() != EpilogueEndI) {
@@ -1349,13 +1371,10 @@ void AArch64EpilogueEmitter::emitEpilogue() {
return;
}
- bool FPAfterSVECalleeSaves =
- Subtarget.isTargetWindows() && AFI->getSVECalleeSavedStackSize();
-
bool CombineSPBump = shouldCombineCSRLocalStackBump(NumBytes);
// Assume we can't combine the last pop with the sp restore.
bool CombineAfterCSRBump = false;
- if (FPAfterSVECalleeSaves) {
+ if (SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord) {
AfterCSRPopSize += FixedObject;
} else if (!CombineSPBump && PrologueSaveSize != 0) {
MachineBasicBlock::iterator Pop = std::prev(MBB.getFirstTerminator());
@@ -1390,7 +1409,8 @@ void AArch64EpilogueEmitter::emitEpilogue() {
while (FirstGPRRestoreI != Begin) {
--FirstGPRRestoreI;
if (!FirstGPRRestoreI->getFlag(MachineInstr::FrameDestroy) ||
- (!FPAfterSVECalleeSaves && isPartOfSVECalleeSaves(FirstGPRRestoreI))) {
+ (SVELayout != SVEStackLayout::CalleeSavesAboveFrameRecord &&
+ isPartOfSVECalleeSaves(FirstGPRRestoreI))) {
++FirstGPRRestoreI;
break;
} else if (CombineSPBump)
@@ -1437,44 +1457,30 @@ void AArch64EpilogueEmitter::emitEpilogue() {
NumBytes -= PrologueSaveSize;
assert(NumBytes >= 0 && "Negative stack allocation size!?");
- if (!AFI->hasSplitSVEObjects()) {
+ if (SVELayout != SVEStackLayout::Split) {
+ StackOffset SVECalleeSavedSize =
+ StackOffset::getScalable(AFI->getSVECalleeSavedStackSize());
+ StackOffset SVELocalsSize = SVEStackSize - SVECalleeSavedSize;
+
// Process the SVE callee-saves to determine what space needs to be
// deallocated.
- StackOffset DeallocateBefore = {}, DeallocateAfter = SVEStackSize;
- MachineBasicBlock::iterator RestoreBegin = FirstGPRRestoreI,
- RestoreEnd = FirstGPRRestoreI;
- int64_t ZPRCalleeSavedSize = AFI->getZPRCalleeSavedStackSize();
- int64_t PPRCalleeSavedSize = AFI->getPPRCalleeSavedStackSize();
- int64_t SVECalleeSavedSize = ZPRCalleeSavedSize + PPRCalleeSavedSize;
-
- if (SVECalleeSavedSize) {
- if (FPAfterSVECalleeSaves)
- RestoreEnd = MBB.getFirstTerminator();
-
- RestoreBegin = std::prev(RestoreEnd);
- while (RestoreBegin != MBB.begin() &&
- isPartOfSVECalleeSaves(std::prev(RestoreBegin)))
- --RestoreBegin;
-
- assert(isPartOfSVECalleeSaves(RestoreBegin) &&
- isPartOfSVECalleeSaves(std::prev(RestoreEnd)) &&
- "Unexpected instruction");
-
- StackOffset CalleeSavedSizeAsOffset =
- StackOffset::getScalable(SVECalleeSavedSize);
- DeallocateBefore = SVEStackSize - CalleeSavedSizeAsOffset;
- DeallocateAfter = CalleeSavedSizeAsOffset;
- }
+ auto [RestoreBegin, RestoreEnd] = partitionSVEEpilogue(
+ MBB,
+ SVECalleeSavedSize &&
+ SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord
+ ? MBB.getFirstTerminator()
+ : FirstGPRRestoreI,
+ SVECalleeSavedSize, SVELocalsSize);
// Deallocate the SVE area.
- if (FPAfterSVECalleeSaves) {
+ if (SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord) {
// If the callee-save area is before FP, restoring the FP implicitly
// deallocates non-callee-save SVE allocations. Otherwise, deallocate
// them explicitly.
if (!AFI->isStackRealigned() && !MFI.hasVarSizedObjects()) {
emitFrameOffset(MBB, FirstGPRRestoreI, DL, AArch64::SP, AArch64::SP,
- DeallocateBefore, TII, MachineInstr::FrameDestroy,
- false, NeedsWinCFI, &HasWinCFI);
+ SVELocalsSize, TII, MachineInstr::FrameDestroy, false,
+ NeedsWinCFI, &HasWinCFI);
}
// Deallocate callee-save non-SVE registers.
@@ -1491,10 +1497,9 @@ void AArch64EpilogueEmitter::emitEpilogue() {
// Deallocate callee-save SVE registers.
emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
- DeallocateAfter, TII, MachineInstr::FrameDestroy, false,
- NeedsWinCFI, &HasWinCFI);
+ SVECalleeSavedSize, TII, MachineInstr::FrameDestroy,
+ false, NeedsWinCFI, &HasWinCFI);
} else if (SVEStackSize) {
- int64_t SVECalleeSavedSize = AFI->getSVECalleeSavedStackSize();
// If we have stack realignment or variable-sized objects we must use the
// FP to restore SVE callee saves (as there is an unknown amount of
// data/padding between the SP and SVE CS area).
@@ -1518,8 +1523,7 @@ void AArch64EpilogueEmitter::emitEpilogue() {
// The code below will deallocate the stack space space by moving the
// SP to the start of the SVE callee-save area.
emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, CalleeSaveBase,
- StackOffset::getScalable(-SVECalleeSavedSize), TII,
- MachineInstr::FrameDestroy);
+ -SVECalleeSavedSize, TII, MachineInstr::FrameDestroy);
} else if (BaseForSVEDealloc == AArch64::SP) {
if (SVECalleeSavedSize) {
// Deallocate the non-SVE locals first before we can deallocate (and
@@ -1534,22 +1538,22 @@ void AArch64EpilogueEmitter::emitEpilogue() {
}
emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
- DeallocateBefore, TII, MachineInstr::FrameDestroy,
- false, NeedsWinCFI, &HasWinCFI, EmitCFI && !HasFP,
+ SVELocalsSize, TII, MachineInstr::FrameDestroy, false,
+ NeedsWinCFI, &HasWinCFI, EmitCFI && !HasFP,
SVEStackSize +
StackOffset::getFixed(NumBytes + PrologueSaveSize));
emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
- DeallocateAfter, TII, MachineInstr::FrameDestroy, false,
- NeedsWinCFI, &HasWinCFI, EmitCFI && !HasFP,
- DeallocateAfter +
+ SVECalleeSavedSize, TII, MachineInstr::FrameDestroy,
+ false, NeedsWinCFI, &HasWinCFI, EmitCFI && !HasFP,
+ SVECalleeSavedSize +
StackOffset::getFixed(NumBytes + PrologueSaveSize));
}
if (EmitCFI)
emitCalleeSavedSVERestores(RestoreEnd);
}
- } else if (AFI->hasSplitSVEObjects() && SVEStackSize) {
+ } else if (SVELayout == SVEStackLayout::Split && SVEStackSize) {
// TODO: Support stack realigment and variable-sized objects.
assert(!AFI->isStackRealigned() && !MFI.hasVarSizedObjects() &&
"unexpected stack realignment or variable sized objects with split "
diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h
index a1c9b34a77c3f..a2a290dd9b7fd 100644
--- a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h
+++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h
@@ -32,6 +32,12 @@ class AArch64PrologueEpilogueCommon {
AArch64PrologueEpilogueCommon(MachineFunction &MF, MachineBasicBlock &MBB,
const AArch64FrameLowering &AFL);
+ enum class SVEStackLayout {
+ Default,
+ Split,
+ CalleeSavesAboveFrameRecord,
+ };
+
protected:
bool requiresGetVGCall() const;
@@ -68,6 +74,7 @@ class AArch64PrologueEpilogueCommon {
bool IsFunclet = false; // Note: Set in derived constructors.
bool NeedsWinCFI = false; // Note: Can be changed in emitFramePointerSetup.
bool HomPrologEpilog = false; // Note: Set in derived constructors.
+ SVEStackLayout SVELayout = SVEStackLayout::Default;
// Note: "HasWinCFI" is mutable as it can change in any "emit" function.
mutable bool HasWinCFI = false;
>From bbfd4637b905732d6871eb626994f6b13c1264d6 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 6 Oct 2025 11:01:12 +0000
Subject: [PATCH 02/13] Step 2
---
.../AArch64/AArch64PrologueEpilogue.cpp | 39 +++++++++++--------
.../Target/AArch64/AArch64PrologueEpilogue.h | 6 +++
2 files changed, 28 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
index 6c24fca272822..74a98f0de837a 100644
--- a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
@@ -354,6 +354,16 @@ bool AArch64PrologueEpilogueCommon::shouldCombineCSRLocalStackBump(
return true;
}
+SVEFrameSizes AArch64PrologueEpilogueCommon::getSVEStackFrameSizes() const {
+ StackOffset PPRCalleeSavesSize =
+ StackOffset::getScalable(AFI->getPPRCalleeSavedStackSize());
+ StackOffset ZPRCalleeSavesSize =
+ StackOffset::getScalable(AFI->getZPRCalleeSavedStackSize());
+ StackOffset PPRLocalsSize = AFL.getPPRStackSize(MF) - PPRCalleeSavesSize;
+ StackOffset ZPRLocalsSize = AFL.getZPRStackSize(MF) - ZPRCalleeSavesSize;
+ return {PPRCalleeSavesSize, ZPRCalleeSavesSize, PPRLocalsSize, ZPRLocalsSize};
+}
+
AArch64PrologueEmitter::AArch64PrologueEmitter(MachineFunction &MF,
MachineBasicBlock &MBB,
const AArch64FrameLowering &AFL)
@@ -714,13 +724,8 @@ void AArch64PrologueEmitter::emitPrologue() {
if (AFL.windowsRequiresStackProbe(MF, NumBytes + RealignmentPadding))
emitWindowsStackProbe(AfterGPRSavesI, DL, NumBytes, RealignmentPadding);
- StackOffset PPRCalleeSavesSize =
- StackOffset::getScalable(AFI->getPPRCalleeSavedStackSize());
- StackOffset ZPRCalleeSavesSize =
- StackOffset::getScalable(AFI->getZPRCalleeSavedStackSize());
- StackOffset SVECalleeSavesSize = PPRCalleeSavesSize + ZPRCalleeSavesSize;
- StackOffset PPRLocalsSize = AFL.getPPRStackSize(MF) - PPRCalleeSavesSize;
- StackOffset ZPRLocalsSize = AFL.getZPRStackSize(MF) - ZPRCalleeSavesSize;
+ auto [PPRCalleeSavesSize, ZPRCalleeSavesSize, PPRLocalsSize, ZPRLocalsSize] =
+ getSVEStackFrameSizes();
std::optional<MachineBasicBlock::iterator> ZPRCalleeSavesBegin,
ZPRCalleeSavesEnd, PPRCalleeSavesBegin, PPRCalleeSavesEnd;
@@ -788,6 +793,7 @@ void AArch64PrologueEmitter::emitPrologue() {
EmitAsyncCFI && !HasFP, CFAOffset,
MFI.hasVarSizedObjects());
} else {
+ StackOffset SVECalleeSavesSize = ZPRCalleeSavesSize + PPRCalleeSavesSize;
// Allocate space for the callee saves (if any).
StackOffset LocalsSize =
PPRLocalsSize + ZPRLocalsSize + StackOffset::getFixed(NumBytes);
@@ -1434,13 +1440,9 @@ void AArch64EpilogueEmitter::emitEpilogue() {
if (HasFP && AFI->hasSwiftAsyncContext())
emitSwiftAsyncContextFramePointer(EpilogueEndI, DL);
- StackOffset ZPRStackSize = AFL.getZPRStackSize(MF);
- StackOffset PPRStackSize = AFL.getPPRStackSize(MF);
- StackOffset SVEStackSize = ZPRStackSize + PPRStackSize;
-
// If there is a single SP update, insert it before the ret and we're done.
if (CombineSPBump) {
- assert(!SVEStackSize && "Cannot combine SP bump with SVE");
+ assert(!AFI->hasSVEStackSize() && "Cannot combine SP bump with SVE");
// When we are about to restore the CSRs, the CFA register is SP again.
if (EmitCFI && HasFP)
@@ -1457,10 +1459,15 @@ void AArch64EpilogueEmitter::emitEpilogue() {
NumBytes -= PrologueSaveSize;
assert(NumBytes >= 0 && "Negative stack allocation size!?");
+ auto [PPRCalleeSavesSize, ZPRCalleeSavesSize, PPRLocalsSize, ZPRLocalsSize] =
+ getSVEStackFrameSizes();
+ StackOffset SVEStackSize =
+ PPRCalleeSavesSize + ZPRCalleeSavesSize + PPRLocalsSize + ZPRLocalsSize;
+
if (SVELayout != SVEStackLayout::Split) {
StackOffset SVECalleeSavedSize =
StackOffset::getScalable(AFI->getSVECalleeSavedStackSize());
- StackOffset SVELocalsSize = SVEStackSize - SVECalleeSavedSize;
+ StackOffset SVELocalsSize = ZPRLocalsSize + PPRLocalsSize;
// Process the SVE callee-saves to determine what space needs to be
// deallocated.
@@ -1499,7 +1506,7 @@ void AArch64EpilogueEmitter::emitEpilogue() {
emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
SVECalleeSavedSize, TII, MachineInstr::FrameDestroy,
false, NeedsWinCFI, &HasWinCFI);
- } else if (SVEStackSize) {
+ } else if (AFI->hasSVEStackSize()) {
// If we have stack realignment or variable-sized objects we must use the
// FP to restore SVE callee saves (as there is an unknown amount of
// data/padding between the SP and SVE CS area).
@@ -1553,7 +1560,7 @@ void AArch64EpilogueEmitter::emitEpilogue() {
if (EmitCFI)
emitCalleeSavedSVERestores(RestoreEnd);
}
- } else if (SVELayout == SVEStackLayout::Split && SVEStackSize) {
+ } else if (SVELayout == SVEStackLayout::Split && AFI->hasSVEStackSize()) {
// TODO: Support stack realigment and variable-sized objects.
assert(!AFI->isStackRealigned() && !MFI.hasVarSizedObjects() &&
"unexpected stack realignment or variable sized objects with split "
@@ -1564,8 +1571,6 @@ void AArch64EpilogueEmitter::emitEpilogue() {
StackOffset::getScalable(AFI->getZPRCalleeSavedStackSize());
auto PPRCalleeSavedSize =
StackOffset::getScalable(AFI->getPPRCalleeSavedStackSize());
- StackOffset PPRLocalsSize = PPRStackSize - PPRCalleeSavedSize;
- StackOffset ZPRLocalsSize = ZPRStackSize - ZPRCalleeSavedSize;
MachineBasicBlock::iterator PPRRestoreBegin = FirstGPRRestoreI,
PPRRestoreEnd = FirstGPRRestoreI;
diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h
index a2a290dd9b7fd..d45ea3e5d57f5 100644
--- a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h
+++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h
@@ -27,6 +27,10 @@ class AArch64Subtarget;
class AArch64FunctionInfo;
class AArch64FrameLowering;
+struct SVEFrameSizes {
+ StackOffset PPRCalleeSavesSize, ZPRCalleeSavesSize;
+ StackOffset PPRLocalsSize, ZPRLocalsSize;
+};
class AArch64PrologueEpilogueCommon {
public:
AArch64PrologueEpilogueCommon(MachineFunction &MF, MachineBasicBlock &MBB,
@@ -59,6 +63,8 @@ class AArch64PrologueEpilogueCommon {
bool shouldCombineCSRLocalStackBump(uint64_t StackBumpBytes) const;
+ SVEFrameSizes getSVEStackFrameSizes() const;
+
MachineFunction &MF;
MachineBasicBlock &MBB;
>From ecfebab8988fc656e8ad960191bd1203bbff0f0b Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 6 Oct 2025 12:00:56 +0000
Subject: [PATCH 03/13] Step 3
---
.../AArch64/AArch64PrologueEpilogue.cpp | 21 +++++++------------
1 file changed, 8 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
index 74a98f0de837a..344371eaa36a3 100644
--- a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
@@ -1567,14 +1567,9 @@ void AArch64EpilogueEmitter::emitEpilogue() {
"SVE stack objects");
// SplitSVEObjects. Determine the sizes and starts/ends of the ZPR and PPR
// areas.
- auto ZPRCalleeSavedSize =
- StackOffset::getScalable(AFI->getZPRCalleeSavedStackSize());
- auto PPRCalleeSavedSize =
- StackOffset::getScalable(AFI->getPPRCalleeSavedStackSize());
-
MachineBasicBlock::iterator PPRRestoreBegin = FirstGPRRestoreI,
PPRRestoreEnd = FirstGPRRestoreI;
- if (PPRCalleeSavedSize) {
+ if (PPRCalleeSavesSize) {
PPRRestoreBegin = std::prev(PPRRestoreEnd);
while (PPRRestoreBegin != MBB.begin() &&
isPartOfPPRCalleeSaves(std::prev(PPRRestoreBegin)))
@@ -1583,7 +1578,7 @@ void AArch64EpilogueEmitter::emitEpilogue() {
MachineBasicBlock::iterator ZPRRestoreBegin = PPRRestoreBegin,
ZPRRestoreEnd = PPRRestoreBegin;
- if (ZPRCalleeSavedSize) {
+ if (ZPRCalleeSavesSize) {
ZPRRestoreBegin = std::prev(ZPRRestoreEnd);
while (ZPRRestoreBegin != MBB.begin() &&
isPartOfZPRCalleeSaves(std::prev(ZPRRestoreBegin)))
@@ -1592,7 +1587,7 @@ void AArch64EpilogueEmitter::emitEpilogue() {
auto CFAOffset =
SVEStackSize + StackOffset::getFixed(NumBytes + PrologueSaveSize);
- if (PPRCalleeSavedSize || ZPRCalleeSavedSize) {
+ if (PPRCalleeSavesSize || ZPRCalleeSavesSize) {
// Deallocate the non-SVE locals first before we can deallocate (and
// restore callee saves) from the SVE area.
auto NonSVELocals = StackOffset::getFixed(NumBytes);
@@ -1610,18 +1605,18 @@ void AArch64EpilogueEmitter::emitEpilogue() {
CFAOffset -= ZPRLocalsSize;
}
- if (PPRLocalsSize || ZPRCalleeSavedSize) {
+ if (PPRLocalsSize || ZPRCalleeSavesSize) {
assert(PPRRestoreBegin == ZPRRestoreEnd &&
"Expected PPR restores after ZPR");
emitFrameOffset(MBB, PPRRestoreBegin, DL, AArch64::SP, AArch64::SP,
- PPRLocalsSize + ZPRCalleeSavedSize, TII,
+ PPRLocalsSize + ZPRCalleeSavesSize, TII,
MachineInstr::FrameDestroy, false, false, nullptr,
EmitCFI && !HasFP, CFAOffset);
- CFAOffset -= PPRLocalsSize + ZPRCalleeSavedSize;
+ CFAOffset -= PPRLocalsSize + ZPRCalleeSavesSize;
}
- if (PPRCalleeSavedSize) {
+ if (PPRCalleeSavesSize) {
emitFrameOffset(MBB, PPRRestoreEnd, DL, AArch64::SP, AArch64::SP,
- PPRCalleeSavedSize, TII, MachineInstr::FrameDestroy,
+ PPRCalleeSavesSize, TII, MachineInstr::FrameDestroy,
false, false, nullptr, EmitCFI && !HasFP, CFAOffset);
}
>From c4b6e359d0ffac2fde9b7fad57e218da150fffed Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 6 Oct 2025 12:11:43 +0000
Subject: [PATCH 04/13] Step 4
---
.../AArch64/AArch64PrologueEpilogue.cpp | 86 +++++++++----------
1 file changed, 40 insertions(+), 46 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
index 344371eaa36a3..30ab81155ae80 100644
--- a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
@@ -1305,25 +1305,34 @@ AArch64EpilogueEmitter::AArch64EpilogueEmitter(MachineFunction &MF,
}
struct SVEEpiloguePartitions {
- MachineBasicBlock::iterator RestoreBegin, RestoreEnd;
+ MachineBasicBlock::iterator PPRRestoreBegin, PPRRestoreEnd;
+ MachineBasicBlock::iterator ZPRRestoreBegin, ZPRRestoreEnd;
+
+ MachineBasicBlock::iterator restoreBegin() { return ZPRRestoreBegin; }
+ MachineBasicBlock::iterator restoreEnd() { return PPRRestoreEnd; }
};
static SVEEpiloguePartitions
partitionSVEEpilogue(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
- StackOffset SVECalleeSavesSize,
- StackOffset SVELocalsSize) {
- MachineBasicBlock::iterator RestoreBegin = MBBI, RestoreEnd = MBBI;
- if (SVECalleeSavesSize) {
- RestoreBegin = std::prev(RestoreEnd);
- while (RestoreBegin != MBB.begin() &&
- isPartOfSVECalleeSaves(std::prev(RestoreBegin)))
- --RestoreBegin;
-
- assert(isPartOfSVECalleeSaves(RestoreBegin) &&
- isPartOfSVECalleeSaves(std::prev(RestoreEnd)) &&
- "Unexpected instruction");
- }
- return {RestoreBegin, RestoreEnd};
+ StackOffset PPRCalleeSavesSize,
+ StackOffset ZPRCalleeSavesSize) {
+ MachineBasicBlock::iterator PPRRestoreBegin = MBBI, PPRRestoreEnd = MBBI;
+ if (PPRCalleeSavesSize) {
+ PPRRestoreBegin = std::prev(PPRRestoreEnd);
+ while (PPRRestoreBegin != MBB.begin() &&
+ isPartOfPPRCalleeSaves(std::prev(PPRRestoreBegin)))
+ --PPRRestoreBegin;
+ }
+ MachineBasicBlock::iterator ZPRRestoreBegin = PPRRestoreBegin,
+ ZPRRestoreEnd = PPRRestoreBegin;
+ if (ZPRCalleeSavesSize) {
+ ZPRRestoreBegin = std::prev(ZPRRestoreEnd);
+ while (ZPRRestoreBegin != MBB.begin() &&
+ isPartOfZPRCalleeSaves(std::prev(ZPRRestoreBegin)))
+ --ZPRRestoreBegin;
+ }
+
+ return {PPRRestoreBegin, PPRRestoreEnd, ZPRRestoreBegin, ZPRRestoreEnd};
}
void AArch64EpilogueEmitter::emitEpilogue() {
@@ -1461,23 +1470,23 @@ void AArch64EpilogueEmitter::emitEpilogue() {
auto [PPRCalleeSavesSize, ZPRCalleeSavesSize, PPRLocalsSize, ZPRLocalsSize] =
getSVEStackFrameSizes();
- StackOffset SVEStackSize =
- PPRCalleeSavesSize + ZPRCalleeSavesSize + PPRLocalsSize + ZPRLocalsSize;
+ StackOffset SVECalleeSavedSize = ZPRCalleeSavesSize + PPRCalleeSavesSize;
+ StackOffset SVEStackSize = SVECalleeSavedSize + PPRLocalsSize + ZPRLocalsSize;
+
+ // Process the SVE callee-saves to determine what space needs to be
+ // deallocated.
+ auto SVEPartitions = partitionSVEEpilogue(
+ MBB,
+ SVECalleeSavedSize &&
+ SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord
+ ? MBB.getFirstTerminator()
+ : FirstGPRRestoreI,
+ PPRCalleeSavesSize, ZPRCalleeSavesSize);
if (SVELayout != SVEStackLayout::Split) {
- StackOffset SVECalleeSavedSize =
- StackOffset::getScalable(AFI->getSVECalleeSavedStackSize());
StackOffset SVELocalsSize = ZPRLocalsSize + PPRLocalsSize;
-
- // Process the SVE callee-saves to determine what space needs to be
- // deallocated.
- auto [RestoreBegin, RestoreEnd] = partitionSVEEpilogue(
- MBB,
- SVECalleeSavedSize &&
- SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord
- ? MBB.getFirstTerminator()
- : FirstGPRRestoreI,
- SVECalleeSavedSize, SVELocalsSize);
+ MachineBasicBlock::iterator RestoreBegin = SVEPartitions.restoreBegin();
+ MachineBasicBlock::iterator RestoreEnd = SVEPartitions.restoreEnd();
// Deallocate the SVE area.
if (SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord) {
@@ -1567,23 +1576,8 @@ void AArch64EpilogueEmitter::emitEpilogue() {
"SVE stack objects");
// SplitSVEObjects. Determine the sizes and starts/ends of the ZPR and PPR
// areas.
- MachineBasicBlock::iterator PPRRestoreBegin = FirstGPRRestoreI,
- PPRRestoreEnd = FirstGPRRestoreI;
- if (PPRCalleeSavesSize) {
- PPRRestoreBegin = std::prev(PPRRestoreEnd);
- while (PPRRestoreBegin != MBB.begin() &&
- isPartOfPPRCalleeSaves(std::prev(PPRRestoreBegin)))
- --PPRRestoreBegin;
- }
-
- MachineBasicBlock::iterator ZPRRestoreBegin = PPRRestoreBegin,
- ZPRRestoreEnd = PPRRestoreBegin;
- if (ZPRCalleeSavesSize) {
- ZPRRestoreBegin = std::prev(ZPRRestoreEnd);
- while (ZPRRestoreBegin != MBB.begin() &&
- isPartOfZPRCalleeSaves(std::prev(ZPRRestoreBegin)))
- --ZPRRestoreBegin;
- }
+ auto [PPRRestoreBegin, PPRRestoreEnd, ZPRRestoreBegin, ZPRRestoreEnd] =
+ SVEPartitions;
auto CFAOffset =
SVEStackSize + StackOffset::getFixed(NumBytes + PrologueSaveSize);
>From 219bf89da446d3af53c7ec677d7999a60a5cac43 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 6 Oct 2025 12:43:52 +0000
Subject: [PATCH 05/13] Step 5
---
llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
index 30ab81155ae80..2ba2cd3c7642f 100644
--- a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
@@ -60,7 +60,6 @@ static bool isPartOfZPRCalleeSaves(MachineBasicBlock::iterator I) {
case AArch64::PTRUE_C_B:
return I->getFlag(MachineInstr::FrameSetup) ||
I->getFlag(MachineInstr::FrameDestroy);
- case AArch64::SEH_SavePReg:
case AArch64::SEH_SaveZReg:
return true;
}
@@ -75,6 +74,8 @@ static bool isPartOfPPRCalleeSaves(MachineBasicBlock::iterator I) {
case AArch64::LDR_PXI:
return I->getFlag(MachineInstr::FrameSetup) ||
I->getFlag(MachineInstr::FrameDestroy);
+ case AArch64::SEH_SavePReg:
+ return true;
}
}
@@ -1475,8 +1476,7 @@ void AArch64EpilogueEmitter::emitEpilogue() {
// Process the SVE callee-saves to determine what space needs to be
// deallocated.
- auto SVEPartitions = partitionSVEEpilogue(
- MBB,
+ auto SVEPartitions = partitionSVEEpilogue(MBB,
SVECalleeSavedSize &&
SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord
? MBB.getFirstTerminator()
>From 3bf181fcec80f17b11bc8524d04f0ab109719eb0 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 6 Oct 2025 14:03:31 +0000
Subject: [PATCH 06/13] Step 6
---
.../AArch64/AArch64PrologueEpilogue.cpp | 118 ++++++++----------
1 file changed, 51 insertions(+), 67 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
index 2ba2cd3c7642f..361844280b4dd 100644
--- a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
@@ -365,6 +365,42 @@ SVEFrameSizes AArch64PrologueEpilogueCommon::getSVEStackFrameSizes() const {
return {PPRCalleeSavesSize, ZPRCalleeSavesSize, PPRLocalsSize, ZPRLocalsSize};
}
+struct SVEPartitions {
+ MachineBasicBlock::iterator PPRBegin, PPREnd;
+ MachineBasicBlock::iterator ZPRBegin, ZPREnd;
+};
+
+static SVEPartitions partitionSVECS(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator MBBI,
+ StackOffset PPRCalleeSavesSize,
+ StackOffset ZPRCalleeSavesSize,
+ bool IsEpilogue) {
+ MachineBasicBlock::iterator PPRsI = MBBI;
+ MachineBasicBlock::iterator End =
+ IsEpilogue ? MBB.begin() : MBB.getFirstTerminator();
+ auto AdjustI = [&](auto MBBI) { return IsEpilogue ? std::prev(MBBI) : MBBI; };
+
+ // Process the SVE callee-saves to find the starts/ends of the ZPR and PPR
+ // areas.
+ if (PPRCalleeSavesSize) {
+ PPRsI = AdjustI(PPRsI);
+ assert(isPartOfPPRCalleeSaves(*PPRsI) && "Unexpected instruction");
+ while (PPRsI != End && isPartOfPPRCalleeSaves(AdjustI(PPRsI)))
+ IsEpilogue ? (--PPRsI) : (++PPRsI);
+ }
+ MachineBasicBlock::iterator ZPRsI = PPRsI;
+ if (ZPRCalleeSavesSize) {
+ ZPRsI = AdjustI(ZPRsI);
+ assert(isPartOfZPRCalleeSaves(*ZPRsI) && "Unexpected instruction");
+ while (ZPRsI != End && isPartOfZPRCalleeSaves(AdjustI(ZPRsI)))
+ IsEpilogue ? (--ZPRsI) : (++ZPRsI);
+ }
+
+ if (IsEpilogue)
+ return {PPRsI, MBBI, ZPRsI, PPRsI};
+ return {MBBI, PPRsI, PPRsI, ZPRsI};
+}
+
AArch64PrologueEmitter::AArch64PrologueEmitter(MachineFunction &MF,
MachineBasicBlock &MBB,
const AArch64FrameLowering &AFL)
@@ -728,37 +764,15 @@ void AArch64PrologueEmitter::emitPrologue() {
auto [PPRCalleeSavesSize, ZPRCalleeSavesSize, PPRLocalsSize, ZPRLocalsSize] =
getSVEStackFrameSizes();
- std::optional<MachineBasicBlock::iterator> ZPRCalleeSavesBegin,
- ZPRCalleeSavesEnd, PPRCalleeSavesBegin, PPRCalleeSavesEnd;
-
StackOffset CFAOffset =
StackOffset::getFixed((int64_t)MFI.getStackSize() - NumBytes);
+
+ SVEPartitions SVECSPartitions;
MachineBasicBlock::iterator AfterSVESavesI = AfterGPRSavesI;
if (SVELayout != SVEStackLayout::CalleeSavesAboveFrameRecord) {
- // Process the SVE callee-saves to find the starts/ends of the ZPR and PPR
- // areas.
- PPRCalleeSavesBegin = AfterGPRSavesI;
- if (PPRCalleeSavesSize) {
- LLVM_DEBUG(dbgs() << "PPRCalleeSavedStackSize = "
- << PPRCalleeSavesSize.getScalable() << "\n");
-
- assert(isPartOfPPRCalleeSaves(*PPRCalleeSavesBegin) &&
- "Unexpected instruction");
- while (isPartOfPPRCalleeSaves(AfterSVESavesI) &&
- AfterSVESavesI != MBB.getFirstTerminator())
- ++AfterSVESavesI;
- }
- PPRCalleeSavesEnd = ZPRCalleeSavesBegin = AfterSVESavesI;
- if (ZPRCalleeSavesSize) {
- LLVM_DEBUG(dbgs() << "ZPRCalleeSavedStackSize = "
- << ZPRCalleeSavesSize.getScalable() << "\n");
- assert(isPartOfZPRCalleeSaves(*ZPRCalleeSavesBegin) &&
- "Unexpected instruction");
- while (isPartOfZPRCalleeSaves(AfterSVESavesI) &&
- AfterSVESavesI != MBB.getFirstTerminator())
- ++AfterSVESavesI;
- }
- ZPRCalleeSavesEnd = AfterSVESavesI;
+ SVECSPartitions = partitionSVECS(MBB, AfterGPRSavesI, PPRCalleeSavesSize,
+ ZPRCalleeSavesSize, /*IsEpilogue=*/false);
+ AfterSVESavesI = SVECSPartitions.ZPREnd;
}
if (EmitAsyncCFI)
@@ -773,23 +787,23 @@ void AArch64PrologueEmitter::emitPrologue() {
// Split ZPR and PPR allocation.
// Allocate PPR callee saves
- allocateStackSpace(*PPRCalleeSavesBegin, 0, PPRCalleeSavesSize,
+ allocateStackSpace(SVECSPartitions.PPRBegin, 0, PPRCalleeSavesSize,
EmitAsyncCFI && !HasFP, CFAOffset,
MFI.hasVarSizedObjects() || ZPRCalleeSavesSize ||
ZPRLocalsSize || PPRLocalsSize);
CFAOffset += PPRCalleeSavesSize;
// Allocate PPR locals + ZPR callee saves
- assert(PPRCalleeSavesEnd == ZPRCalleeSavesBegin &&
+ assert(SVECSPartitions.PPREnd == SVECSPartitions.ZPRBegin &&
"Expected ZPR callee saves after PPR locals");
- allocateStackSpace(*PPRCalleeSavesEnd, RealignmentPadding,
+ allocateStackSpace(SVECSPartitions.PPREnd, RealignmentPadding,
PPRLocalsSize + ZPRCalleeSavesSize,
EmitAsyncCFI && !HasFP, CFAOffset,
MFI.hasVarSizedObjects() || ZPRLocalsSize);
CFAOffset += PPRLocalsSize + ZPRCalleeSavesSize;
// Allocate ZPR locals
- allocateStackSpace(*ZPRCalleeSavesEnd, RealignmentPadding,
+ allocateStackSpace(SVECSPartitions.ZPREnd, RealignmentPadding,
ZPRLocalsSize + StackOffset::getFixed(NumBytes),
EmitAsyncCFI && !HasFP, CFAOffset,
MFI.hasVarSizedObjects());
@@ -1305,37 +1319,6 @@ AArch64EpilogueEmitter::AArch64EpilogueEmitter(MachineFunction &MF,
SEHEpilogueStartI = MBB.end();
}
-struct SVEEpiloguePartitions {
- MachineBasicBlock::iterator PPRRestoreBegin, PPRRestoreEnd;
- MachineBasicBlock::iterator ZPRRestoreBegin, ZPRRestoreEnd;
-
- MachineBasicBlock::iterator restoreBegin() { return ZPRRestoreBegin; }
- MachineBasicBlock::iterator restoreEnd() { return PPRRestoreEnd; }
-};
-
-static SVEEpiloguePartitions
-partitionSVEEpilogue(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
- StackOffset PPRCalleeSavesSize,
- StackOffset ZPRCalleeSavesSize) {
- MachineBasicBlock::iterator PPRRestoreBegin = MBBI, PPRRestoreEnd = MBBI;
- if (PPRCalleeSavesSize) {
- PPRRestoreBegin = std::prev(PPRRestoreEnd);
- while (PPRRestoreBegin != MBB.begin() &&
- isPartOfPPRCalleeSaves(std::prev(PPRRestoreBegin)))
- --PPRRestoreBegin;
- }
- MachineBasicBlock::iterator ZPRRestoreBegin = PPRRestoreBegin,
- ZPRRestoreEnd = PPRRestoreBegin;
- if (ZPRCalleeSavesSize) {
- ZPRRestoreBegin = std::prev(ZPRRestoreEnd);
- while (ZPRRestoreBegin != MBB.begin() &&
- isPartOfZPRCalleeSaves(std::prev(ZPRRestoreBegin)))
- --ZPRRestoreBegin;
- }
-
- return {PPRRestoreBegin, PPRRestoreEnd, ZPRRestoreBegin, ZPRRestoreEnd};
-}
-
void AArch64EpilogueEmitter::emitEpilogue() {
MachineBasicBlock::iterator EpilogueEndI = MBB.getLastNonDebugInstr();
if (MBB.end() != EpilogueEndI) {
@@ -1476,17 +1459,18 @@ void AArch64EpilogueEmitter::emitEpilogue() {
// Process the SVE callee-saves to determine what space needs to be
// deallocated.
- auto SVEPartitions = partitionSVEEpilogue(MBB,
+ auto SVECSPartitions = partitionSVECS(
+ MBB,
SVECalleeSavedSize &&
SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord
? MBB.getFirstTerminator()
: FirstGPRRestoreI,
- PPRCalleeSavesSize, ZPRCalleeSavesSize);
+ PPRCalleeSavesSize, ZPRCalleeSavesSize, /*IsEpilogue=*/true);
if (SVELayout != SVEStackLayout::Split) {
StackOffset SVELocalsSize = ZPRLocalsSize + PPRLocalsSize;
- MachineBasicBlock::iterator RestoreBegin = SVEPartitions.restoreBegin();
- MachineBasicBlock::iterator RestoreEnd = SVEPartitions.restoreEnd();
+ MachineBasicBlock::iterator RestoreBegin = SVECSPartitions.ZPRBegin;
+ MachineBasicBlock::iterator RestoreEnd = SVECSPartitions.PPREnd;
// Deallocate the SVE area.
if (SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord) {
@@ -1577,7 +1561,7 @@ void AArch64EpilogueEmitter::emitEpilogue() {
// SplitSVEObjects. Determine the sizes and starts/ends of the ZPR and PPR
// areas.
auto [PPRRestoreBegin, PPRRestoreEnd, ZPRRestoreBegin, ZPRRestoreEnd] =
- SVEPartitions;
+ SVECSPartitions;
auto CFAOffset =
SVEStackSize + StackOffset::getFixed(NumBytes + PrologueSaveSize);
>From c1012cae6a9baa0a3dbf94786a17293e05375845 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 6 Oct 2025 15:14:56 +0000
Subject: [PATCH 07/13] Step 7
---
.../AArch64/AArch64PrologueEpilogue.cpp | 228 ++++++++----------
1 file changed, 102 insertions(+), 126 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
index 361844280b4dd..97c4cfff952d6 100644
--- a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
@@ -362,7 +362,12 @@ SVEFrameSizes AArch64PrologueEpilogueCommon::getSVEStackFrameSizes() const {
StackOffset::getScalable(AFI->getZPRCalleeSavedStackSize());
StackOffset PPRLocalsSize = AFL.getPPRStackSize(MF) - PPRCalleeSavesSize;
StackOffset ZPRLocalsSize = AFL.getZPRStackSize(MF) - ZPRCalleeSavesSize;
- return {PPRCalleeSavesSize, ZPRCalleeSavesSize, PPRLocalsSize, ZPRLocalsSize};
+ if (AFI->hasSplitSVEObjects())
+ return {PPRCalleeSavesSize, ZPRCalleeSavesSize, PPRLocalsSize,
+ ZPRLocalsSize};
+ // For simplicity, attribute all locals to ZPRs when split SVE is disabled.
+ return {PPRCalleeSavesSize, ZPRCalleeSavesSize, StackOffset{},
+ PPRLocalsSize + ZPRLocalsSize};
}
struct SVEPartitions {
@@ -1457,8 +1462,6 @@ void AArch64EpilogueEmitter::emitEpilogue() {
StackOffset SVECalleeSavedSize = ZPRCalleeSavesSize + PPRCalleeSavesSize;
StackOffset SVEStackSize = SVECalleeSavedSize + PPRLocalsSize + ZPRLocalsSize;
- // Process the SVE callee-saves to determine what space needs to be
- // deallocated.
auto SVECSPartitions = partitionSVECS(
MBB,
SVECalleeSavedSize &&
@@ -1467,140 +1470,113 @@ void AArch64EpilogueEmitter::emitEpilogue() {
: FirstGPRRestoreI,
PPRCalleeSavesSize, ZPRCalleeSavesSize, /*IsEpilogue=*/true);
- if (SVELayout != SVEStackLayout::Split) {
+ MachineBasicBlock::iterator RestoreBegin = SVECSPartitions.ZPRBegin;
+ MachineBasicBlock::iterator RestoreEnd = SVECSPartitions.PPREnd;
+
+ // Deallocate the SVE area.
+ if (SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord) {
StackOffset SVELocalsSize = ZPRLocalsSize + PPRLocalsSize;
- MachineBasicBlock::iterator RestoreBegin = SVECSPartitions.ZPRBegin;
- MachineBasicBlock::iterator RestoreEnd = SVECSPartitions.PPREnd;
-
- // Deallocate the SVE area.
- if (SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord) {
- // If the callee-save area is before FP, restoring the FP implicitly
- // deallocates non-callee-save SVE allocations. Otherwise, deallocate
- // them explicitly.
- if (!AFI->isStackRealigned() && !MFI.hasVarSizedObjects()) {
- emitFrameOffset(MBB, FirstGPRRestoreI, DL, AArch64::SP, AArch64::SP,
- SVELocalsSize, TII, MachineInstr::FrameDestroy, false,
- NeedsWinCFI, &HasWinCFI);
- }
+ // If the callee-save area is before FP, restoring the FP implicitly
+ // deallocates non-callee-save SVE allocations. Otherwise, deallocate them
+ // explicitly.
+ if (!AFI->isStackRealigned() && !MFI.hasVarSizedObjects()) {
+ emitFrameOffset(MBB, FirstGPRRestoreI, DL, AArch64::SP, AArch64::SP,
+ SVELocalsSize, TII, MachineInstr::FrameDestroy, false,
+ NeedsWinCFI, &HasWinCFI);
+ }
- // Deallocate callee-save non-SVE registers.
- emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
- StackOffset::getFixed(AFI->getCalleeSavedStackSize()),
- TII, MachineInstr::FrameDestroy, false, NeedsWinCFI,
- &HasWinCFI);
-
- // Deallocate fixed objects.
- emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
- StackOffset::getFixed(FixedObject), TII,
- MachineInstr::FrameDestroy, false, NeedsWinCFI,
- &HasWinCFI);
-
- // Deallocate callee-save SVE registers.
- emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
- SVECalleeSavedSize, TII, MachineInstr::FrameDestroy,
- false, NeedsWinCFI, &HasWinCFI);
- } else if (AFI->hasSVEStackSize()) {
- // If we have stack realignment or variable-sized objects we must use the
- // FP to restore SVE callee saves (as there is an unknown amount of
- // data/padding between the SP and SVE CS area).
- Register BaseForSVEDealloc =
- (AFI->isStackRealigned() || MFI.hasVarSizedObjects()) ? AArch64::FP
- : AArch64::SP;
- if (SVECalleeSavedSize && BaseForSVEDealloc == AArch64::FP) {
- Register CalleeSaveBase = AArch64::FP;
- if (int64_t CalleeSaveBaseOffset =
- AFI->getCalleeSaveBaseToFrameRecordOffset()) {
- // If we have have an non-zero offset to the non-SVE CS base we need
- // to compute the base address by subtracting the offest in a
- // temporary register first (to avoid briefly deallocating the SVE
- // CS).
- CalleeSaveBase = MBB.getParent()->getRegInfo().createVirtualRegister(
- &AArch64::GPR64RegClass);
- emitFrameOffset(MBB, RestoreBegin, DL, CalleeSaveBase, AArch64::FP,
- StackOffset::getFixed(-CalleeSaveBaseOffset), TII,
- MachineInstr::FrameDestroy);
- }
- // The code below will deallocate the stack space space by moving the
- // SP to the start of the SVE callee-save area.
- emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, CalleeSaveBase,
- -SVECalleeSavedSize, TII, MachineInstr::FrameDestroy);
- } else if (BaseForSVEDealloc == AArch64::SP) {
- if (SVECalleeSavedSize) {
- // Deallocate the non-SVE locals first before we can deallocate (and
- // restore callee saves) from the SVE area.
- emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
- StackOffset::getFixed(NumBytes), TII,
- MachineInstr::FrameDestroy, false, NeedsWinCFI,
- &HasWinCFI, EmitCFI && !HasFP,
- SVEStackSize + StackOffset::getFixed(
- NumBytes + PrologueSaveSize));
- NumBytes = 0;
- }
-
- emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
- SVELocalsSize, TII, MachineInstr::FrameDestroy, false,
- NeedsWinCFI, &HasWinCFI, EmitCFI && !HasFP,
- SVEStackSize +
- StackOffset::getFixed(NumBytes + PrologueSaveSize));
-
- emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
- SVECalleeSavedSize, TII, MachineInstr::FrameDestroy,
- false, NeedsWinCFI, &HasWinCFI, EmitCFI && !HasFP,
- SVECalleeSavedSize +
- StackOffset::getFixed(NumBytes + PrologueSaveSize));
- }
+ // Deallocate callee-save non-SVE registers.
+ emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::SP,
+ StackOffset::getFixed(AFI->getCalleeSavedStackSize()), TII,
+ MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
- if (EmitCFI)
- emitCalleeSavedSVERestores(RestoreEnd);
- }
- } else if (SVELayout == SVEStackLayout::Split && AFI->hasSVEStackSize()) {
- // TODO: Support stack realigment and variable-sized objects.
- assert(!AFI->isStackRealigned() && !MFI.hasVarSizedObjects() &&
- "unexpected stack realignment or variable sized objects with split "
- "SVE stack objects");
- // SplitSVEObjects. Determine the sizes and starts/ends of the ZPR and PPR
- // areas.
+ // Deallocate fixed objects.
+ emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
+ StackOffset::getFixed(FixedObject), TII,
+ MachineInstr::FrameDestroy, false, NeedsWinCFI, &HasWinCFI);
+
+ // Deallocate callee-save SVE registers.
+ emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
+ SVECalleeSavedSize, TII, MachineInstr::FrameDestroy, false,
+ NeedsWinCFI, &HasWinCFI);
+ } else if (AFI->hasSVEStackSize()) {
auto [PPRRestoreBegin, PPRRestoreEnd, ZPRRestoreBegin, ZPRRestoreEnd] =
SVECSPartitions;
+ // If we have stack realignment or variable-sized objects we must use the FP
+ // to restore SVE callee saves (as there is an unknown amount of
+ // data/padding between the SP and SVE CS area).
+ Register BaseForSVEDealloc =
+ (AFI->isStackRealigned() || MFI.hasVarSizedObjects()) ? AArch64::FP
+ : AArch64::SP;
+ if (SVECalleeSavedSize && BaseForSVEDealloc == AArch64::FP) {
+ // TODO: Support stack realigment and variable-sized objects.
+ assert(
+ !AFI->hasSplitSVEObjects() &&
+ "unexpected stack realignment or variable sized objects with split "
+ "SVE stack objects");
+
+ Register CalleeSaveBase = AArch64::FP;
+ if (int64_t CalleeSaveBaseOffset =
+ AFI->getCalleeSaveBaseToFrameRecordOffset()) {
+ // If we have have an non-zero offset to the non-SVE CS base we need to
+ // compute the base address by subtracting the offest in a temporary
+ // register first (to avoid briefly deallocating the SVE CS).
+ CalleeSaveBase = MBB.getParent()->getRegInfo().createVirtualRegister(
+ &AArch64::GPR64RegClass);
+ emitFrameOffset(MBB, RestoreBegin, DL, CalleeSaveBase, AArch64::FP,
+ StackOffset::getFixed(-CalleeSaveBaseOffset), TII,
+ MachineInstr::FrameDestroy);
+ }
+ // The code below will deallocate the stack space space by moving the SP
+ // to the start of the SVE callee-save area.
+ emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, CalleeSaveBase,
+ -SVECalleeSavedSize, TII, MachineInstr::FrameDestroy);
+ } else if (BaseForSVEDealloc == AArch64::SP) {
+ auto CFAOffset =
+ SVEStackSize + StackOffset::getFixed(NumBytes + PrologueSaveSize);
+
+ if (SVECalleeSavedSize) {
+ // Deallocate the non-SVE locals first before we can deallocate (and
+ // restore callee saves) from the SVE area.
+ auto NonSVELocals = StackOffset::getFixed(NumBytes);
+ emitFrameOffset(MBB, ZPRRestoreBegin, DL, AArch64::SP, AArch64::SP,
+ NonSVELocals, TII, MachineInstr::FrameDestroy, false,
+ NeedsWinCFI, &HasWinCFI, EmitCFI && !HasFP, CFAOffset);
+ CFAOffset -= NonSVELocals;
+ NumBytes = 0;
+ }
- auto CFAOffset =
- SVEStackSize + StackOffset::getFixed(NumBytes + PrologueSaveSize);
- if (PPRCalleeSavesSize || ZPRCalleeSavesSize) {
- // Deallocate the non-SVE locals first before we can deallocate (and
- // restore callee saves) from the SVE area.
- auto NonSVELocals = StackOffset::getFixed(NumBytes);
- emitFrameOffset(MBB, ZPRRestoreBegin, DL, AArch64::SP, AArch64::SP,
- NonSVELocals, TII, MachineInstr::FrameDestroy, false,
- false, nullptr, EmitCFI && !HasFP, CFAOffset);
- NumBytes = 0;
- CFAOffset -= NonSVELocals;
- }
+ if (ZPRLocalsSize) {
+ emitFrameOffset(MBB, ZPRRestoreBegin, DL, AArch64::SP, AArch64::SP,
+ ZPRLocalsSize, TII, MachineInstr::FrameDestroy, false,
+ NeedsWinCFI, &HasWinCFI, EmitCFI && !HasFP, CFAOffset);
+ CFAOffset -= ZPRLocalsSize;
+ }
- if (ZPRLocalsSize) {
- emitFrameOffset(MBB, ZPRRestoreBegin, DL, AArch64::SP, AArch64::SP,
- ZPRLocalsSize, TII, MachineInstr::FrameDestroy, false,
- false, nullptr, EmitCFI && !HasFP, CFAOffset);
- CFAOffset -= ZPRLocalsSize;
- }
+ StackOffset SVECalleeSavesToDealloc = SVECalleeSavedSize;
+ if (SVELayout == SVEStackLayout::Split &&
+ (PPRLocalsSize || ZPRCalleeSavesSize)) {
+ assert(PPRRestoreBegin == ZPRRestoreEnd &&
+ "Expected PPR restores after ZPR");
+ emitFrameOffset(MBB, PPRRestoreBegin, DL, AArch64::SP, AArch64::SP,
+ PPRLocalsSize + ZPRCalleeSavesSize, TII,
+ MachineInstr::FrameDestroy, false, NeedsWinCFI,
+ &HasWinCFI, EmitCFI && !HasFP, CFAOffset);
+ CFAOffset -= PPRLocalsSize + ZPRCalleeSavesSize;
+ SVECalleeSavesToDealloc -= ZPRCalleeSavesSize;
+ }
- if (PPRLocalsSize || ZPRCalleeSavesSize) {
- assert(PPRRestoreBegin == ZPRRestoreEnd &&
- "Expected PPR restores after ZPR");
- emitFrameOffset(MBB, PPRRestoreBegin, DL, AArch64::SP, AArch64::SP,
- PPRLocalsSize + ZPRCalleeSavesSize, TII,
- MachineInstr::FrameDestroy, false, false, nullptr,
- EmitCFI && !HasFP, CFAOffset);
- CFAOffset -= PPRLocalsSize + ZPRCalleeSavesSize;
- }
- if (PPRCalleeSavesSize) {
- emitFrameOffset(MBB, PPRRestoreEnd, DL, AArch64::SP, AArch64::SP,
- PPRCalleeSavesSize, TII, MachineInstr::FrameDestroy,
- false, false, nullptr, EmitCFI && !HasFP, CFAOffset);
+ // If split SVE is on, this dealloc PPRs, otherwise, deallocs ZPRs + PPRs:
+ if (SVECalleeSavesToDealloc)
+ emitFrameOffset(MBB, PPRRestoreEnd, DL, AArch64::SP, AArch64::SP,
+ SVECalleeSavesToDealloc, TII,
+ MachineInstr::FrameDestroy, false, NeedsWinCFI,
+ &HasWinCFI, EmitCFI && !HasFP, CFAOffset);
}
- // We only emit CFI information for ZPRs so emit CFI after the ZPR restores.
if (EmitCFI)
- emitCalleeSavedSVERestores(ZPRRestoreEnd);
+ emitCalleeSavedSVERestores(
+ SVELayout == SVEStackLayout::Split ? ZPRRestoreEnd : PPRRestoreEnd);
}
if (!HasFP) {
>From 720ee895739c9b9854b433c5b8f9b8a3de36896e Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 6 Oct 2025 16:23:24 +0000
Subject: [PATCH 08/13] Step 8
---
.../AArch64/AArch64PrologueEpilogue.cpp | 122 ++++++++----------
.../Target/AArch64/AArch64PrologueEpilogue.h | 1 +
2 files changed, 53 insertions(+), 70 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
index 97c4cfff952d6..00468769d3896 100644
--- a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
@@ -384,9 +384,7 @@ static SVEPartitions partitionSVECS(MachineBasicBlock &MBB,
MachineBasicBlock::iterator End =
IsEpilogue ? MBB.begin() : MBB.getFirstTerminator();
auto AdjustI = [&](auto MBBI) { return IsEpilogue ? std::prev(MBBI) : MBBI; };
-
- // Process the SVE callee-saves to find the starts/ends of the ZPR and PPR
- // areas.
+ // Process the SVE CS to find the starts/ends of the ZPR and PPR areas.
if (PPRCalleeSavesSize) {
PPRsI = AdjustI(PPRsI);
assert(isPartOfPPRCalleeSaves(*PPRsI) && "Unexpected instruction");
@@ -400,7 +398,6 @@ static SVEPartitions partitionSVECS(MachineBasicBlock &MBB,
while (ZPRsI != End && isPartOfZPRCalleeSaves(AdjustI(ZPRsI)))
IsEpilogue ? (--ZPRsI) : (++ZPRsI);
}
-
if (IsEpilogue)
return {PPRsI, MBBI, ZPRsI, PPRsI};
return {MBBI, PPRsI, PPRsI, ZPRsI};
@@ -766,82 +763,67 @@ void AArch64PrologueEmitter::emitPrologue() {
if (AFL.windowsRequiresStackProbe(MF, NumBytes + RealignmentPadding))
emitWindowsStackProbe(AfterGPRSavesI, DL, NumBytes, RealignmentPadding);
+ assert(!(AFL.canUseRedZone(MF) && NeedsRealignment) &&
+ "Cannot use redzone with stack realignment");
+
auto [PPRCalleeSavesSize, ZPRCalleeSavesSize, PPRLocalsSize, ZPRLocalsSize] =
getSVEStackFrameSizes();
+ StackOffset NonSVELocalsSize = StackOffset::getFixed(NumBytes);
+ StackOffset SVECalleeSavesSize = ZPRCalleeSavesSize + PPRCalleeSavesSize;
StackOffset CFAOffset =
- StackOffset::getFixed((int64_t)MFI.getStackSize() - NumBytes);
+ StackOffset::getFixed(MFI.getStackSize()) - NonSVELocalsSize;
- SVEPartitions SVECSPartitions;
MachineBasicBlock::iterator AfterSVESavesI = AfterGPRSavesI;
+ // Allocate space for the callee saves and PPR locals (if any).
if (SVELayout != SVEStackLayout::CalleeSavesAboveFrameRecord) {
- SVECSPartitions = partitionSVECS(MBB, AfterGPRSavesI, PPRCalleeSavesSize,
- ZPRCalleeSavesSize, /*IsEpilogue=*/false);
+ SVEPartitions SVECSPartitions =
+ partitionSVECS(MBB, AfterGPRSavesI, PPRCalleeSavesSize,
+ ZPRCalleeSavesSize, /*IsEpilogue=*/false);
AfterSVESavesI = SVECSPartitions.ZPREnd;
- }
-
- if (EmitAsyncCFI)
- emitCalleeSavedSVELocations(AfterSVESavesI);
-
- if (SVELayout == SVEStackLayout::Split) {
- assert(!AFL.canUseRedZone(MF) &&
- "Cannot use redzone with aarch64-split-sve-objects");
- // TODO: Handle HasWinCFI/NeedsWinCFI?
- assert(!NeedsWinCFI &&
- "WinCFI with aarch64-split-sve-objects is not supported");
-
- // Split ZPR and PPR allocation.
- // Allocate PPR callee saves
- allocateStackSpace(SVECSPartitions.PPRBegin, 0, PPRCalleeSavesSize,
+ if (EmitAsyncCFI)
+ emitCalleeSavedSVELocations(AfterSVESavesI);
+
+ StackOffset AllocateBeforePPRs = SVECalleeSavesSize;
+ StackOffset AllocateAfterPPRs = PPRLocalsSize;
+ if (SVELayout == SVEStackLayout::Split) {
+ AllocateBeforePPRs = PPRCalleeSavesSize;
+ AllocateAfterPPRs = PPRLocalsSize + ZPRCalleeSavesSize;
+ }
+ allocateStackSpace(SVECSPartitions.PPRBegin, 0, AllocateBeforePPRs,
EmitAsyncCFI && !HasFP, CFAOffset,
- MFI.hasVarSizedObjects() || ZPRCalleeSavesSize ||
- ZPRLocalsSize || PPRLocalsSize);
- CFAOffset += PPRCalleeSavesSize;
-
- // Allocate PPR locals + ZPR callee saves
+ MFI.hasVarSizedObjects() || AllocateAfterPPRs ||
+ ZPRLocalsSize || NonSVELocalsSize);
+ CFAOffset += AllocateBeforePPRs;
assert(SVECSPartitions.PPREnd == SVECSPartitions.ZPRBegin &&
"Expected ZPR callee saves after PPR locals");
allocateStackSpace(SVECSPartitions.PPREnd, RealignmentPadding,
- PPRLocalsSize + ZPRCalleeSavesSize,
- EmitAsyncCFI && !HasFP, CFAOffset,
- MFI.hasVarSizedObjects() || ZPRLocalsSize);
- CFAOffset += PPRLocalsSize + ZPRCalleeSavesSize;
-
- // Allocate ZPR locals
- allocateStackSpace(SVECSPartitions.ZPREnd, RealignmentPadding,
- ZPRLocalsSize + StackOffset::getFixed(NumBytes),
- EmitAsyncCFI && !HasFP, CFAOffset,
- MFI.hasVarSizedObjects());
+ AllocateAfterPPRs, EmitAsyncCFI && !HasFP, CFAOffset,
+ MFI.hasVarSizedObjects() || ZPRLocalsSize ||
+ NonSVELocalsSize);
+ CFAOffset += AllocateAfterPPRs;
} else {
- StackOffset SVECalleeSavesSize = ZPRCalleeSavesSize + PPRCalleeSavesSize;
- // Allocate space for the callee saves (if any).
- StackOffset LocalsSize =
- PPRLocalsSize + ZPRLocalsSize + StackOffset::getFixed(NumBytes);
- if (SVELayout != SVEStackLayout::CalleeSavesAboveFrameRecord)
- allocateStackSpace(AfterGPRSavesI, 0, SVECalleeSavesSize,
- EmitAsyncCFI && !HasFP, CFAOffset,
- MFI.hasVarSizedObjects() || LocalsSize);
+ // Note: With CalleeSavesAboveFrameRecord the SVE CS have already been
+ // allocated (and separate PPRLocals are not supported).
+ assert(!PPRLocalsSize && "Unexpected PPR locals!");
CFAOffset += SVECalleeSavesSize;
+ }
- // Allocate space for the rest of the frame including SVE locals. Align the
- // stack as necessary.
- assert(!(AFL.canUseRedZone(MF) && NeedsRealignment) &&
- "Cannot use redzone with stack realignment");
- if (!AFL.canUseRedZone(MF)) {
- // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have
- // the correct value here, as NumBytes also includes padding bytes,
- // which shouldn't be counted here.
- StackOffset SVELocalsSize = PPRLocalsSize + ZPRLocalsSize;
- allocateStackSpace(AfterSVESavesI, RealignmentPadding,
- SVELocalsSize + StackOffset::getFixed(NumBytes),
- EmitAsyncCFI && !HasFP, CFAOffset,
- MFI.hasVarSizedObjects());
- }
+ // Allocate space for the rest of the frame including ZPR locals. Align the
+ // stack as necessary.
+ if (!AFL.canUseRedZone(MF)) {
+ // FIXME: in the case of dynamic re-alignment, NumBytes doesn't have the
+ // correct value here, as NumBytes also includes padding bytes, which
+ // shouldn't be counted here.
+ allocateStackSpace(AfterSVESavesI, RealignmentPadding,
+ ZPRLocalsSize + NonSVELocalsSize, EmitAsyncCFI && !HasFP,
+ CFAOffset, MFI.hasVarSizedObjects());
}
// If we need a base pointer, set it up here. It's whatever the value of the
- // stack pointer is at this point. Any variable size objects will be allocated
- // after this, so we can still use the base pointer to reference locals.
+ // stack pointer is at this point. Any variable size objects will be
+ // allocated after this, so we can still use the base pointer to reference
+ // locals.
//
// FIXME: Clarify FrameSetup flags here.
// Note: Use emitFrameOffset() like above for FP if the FrameSetup flag is
@@ -1459,12 +1441,12 @@ void AArch64EpilogueEmitter::emitEpilogue() {
auto [PPRCalleeSavesSize, ZPRCalleeSavesSize, PPRLocalsSize, ZPRLocalsSize] =
getSVEStackFrameSizes();
- StackOffset SVECalleeSavedSize = ZPRCalleeSavesSize + PPRCalleeSavesSize;
- StackOffset SVEStackSize = SVECalleeSavedSize + PPRLocalsSize + ZPRLocalsSize;
+ StackOffset SVECalleeSavesSize = ZPRCalleeSavesSize + PPRCalleeSavesSize;
+ StackOffset SVEStackSize = SVECalleeSavesSize + PPRLocalsSize + ZPRLocalsSize;
auto SVECSPartitions = partitionSVECS(
MBB,
- SVECalleeSavedSize &&
+ SVECalleeSavesSize &&
SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord
? MBB.getFirstTerminator()
: FirstGPRRestoreI,
@@ -1497,7 +1479,7 @@ void AArch64EpilogueEmitter::emitEpilogue() {
// Deallocate callee-save SVE registers.
emitFrameOffset(MBB, RestoreEnd, DL, AArch64::SP, AArch64::SP,
- SVECalleeSavedSize, TII, MachineInstr::FrameDestroy, false,
+ SVECalleeSavesSize, TII, MachineInstr::FrameDestroy, false,
NeedsWinCFI, &HasWinCFI);
} else if (AFI->hasSVEStackSize()) {
auto [PPRRestoreBegin, PPRRestoreEnd, ZPRRestoreBegin, ZPRRestoreEnd] =
@@ -1508,7 +1490,7 @@ void AArch64EpilogueEmitter::emitEpilogue() {
Register BaseForSVEDealloc =
(AFI->isStackRealigned() || MFI.hasVarSizedObjects()) ? AArch64::FP
: AArch64::SP;
- if (SVECalleeSavedSize && BaseForSVEDealloc == AArch64::FP) {
+ if (SVECalleeSavesSize && BaseForSVEDealloc == AArch64::FP) {
// TODO: Support stack realigment and variable-sized objects.
assert(
!AFI->hasSplitSVEObjects() &&
@@ -1530,12 +1512,12 @@ void AArch64EpilogueEmitter::emitEpilogue() {
// The code below will deallocate the stack space space by moving the SP
// to the start of the SVE callee-save area.
emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, CalleeSaveBase,
- -SVECalleeSavedSize, TII, MachineInstr::FrameDestroy);
+ -SVECalleeSavesSize, TII, MachineInstr::FrameDestroy);
} else if (BaseForSVEDealloc == AArch64::SP) {
auto CFAOffset =
SVEStackSize + StackOffset::getFixed(NumBytes + PrologueSaveSize);
- if (SVECalleeSavedSize) {
+ if (SVECalleeSavesSize) {
// Deallocate the non-SVE locals first before we can deallocate (and
// restore callee saves) from the SVE area.
auto NonSVELocals = StackOffset::getFixed(NumBytes);
@@ -1553,7 +1535,7 @@ void AArch64EpilogueEmitter::emitEpilogue() {
CFAOffset -= ZPRLocalsSize;
}
- StackOffset SVECalleeSavesToDealloc = SVECalleeSavedSize;
+ StackOffset SVECalleeSavesToDealloc = SVECalleeSavesSize;
if (SVELayout == SVEStackLayout::Split &&
(PPRLocalsSize || ZPRCalleeSavesSize)) {
assert(PPRRestoreBegin == ZPRRestoreEnd &&
diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h
index d45ea3e5d57f5..4347b28600b5e 100644
--- a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h
+++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h
@@ -31,6 +31,7 @@ struct SVEFrameSizes {
StackOffset PPRCalleeSavesSize, ZPRCalleeSavesSize;
StackOffset PPRLocalsSize, ZPRLocalsSize;
};
+
class AArch64PrologueEpilogueCommon {
public:
AArch64PrologueEpilogueCommon(MachineFunction &MF, MachineBasicBlock &MBB,
>From cc335258747f62d68e0970413a385b8e5e28f6a6 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 6 Oct 2025 16:37:44 +0000
Subject: [PATCH 09/13] Format
---
llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
index 00468769d3896..583aa2f762148 100644
--- a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
@@ -97,11 +97,11 @@ AArch64PrologueEpilogueCommon::AArch64PrologueEpilogueCommon(
NeedsWinCFI = AFL.needsWinCFI(MF);
// Windows unwind can't represent the required stack adjustments if we have
- // both SVE callee-saves and dynamic stack allocations, and the frame
- // pointer is before the SVE spills. The allocation of the frame pointer
- // must be the last instruction in the prologue so the unwinder can restore
- // the stack pointer correctly. (And there isn't any unwind opcode for
- // `addvl sp, x29, -17`.)
+ // both SVE callee-saves and dynamic stack allocations, and the frame pointer
+ // is before the SVE spills. The allocation of the frame pointer must be the
+ // last instruction in the prologue so the unwinder can restore the stack
+ // pointer correctly. (And there isn't any unwind opcode for `addvl sp, x29,
+ // -17`.)
//
// Because of this, we do spills in the opposite order on Windows: first SVE,
// then GPRs. The main side-effect of this is that it makes accessing
>From 3d178c098d3f3ab9c573ab82103baca8428f7bd7 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Mon, 6 Oct 2025 16:48:16 +0000
Subject: [PATCH 10/13] Remove redundant check
---
llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
index 583aa2f762148..39d7ec3a9559d 100644
--- a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
@@ -1446,8 +1446,7 @@ void AArch64EpilogueEmitter::emitEpilogue() {
auto SVECSPartitions = partitionSVECS(
MBB,
- SVECalleeSavesSize &&
- SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord
+ SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord
? MBB.getFirstTerminator()
: FirstGPRRestoreI,
PPRCalleeSavesSize, ZPRCalleeSavesSize, /*IsEpilogue=*/true);
>From 42d08e63d32a76aca507361ab2fdbd76ccb4f578 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Tue, 7 Oct 2025 09:29:22 +0000
Subject: [PATCH 11/13] Tweak naming
---
.../AArch64/AArch64PrologueEpilogue.cpp | 107 +++++++++---------
.../Target/AArch64/AArch64PrologueEpilogue.h | 5 +-
2 files changed, 55 insertions(+), 57 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
index 39d7ec3a9559d..db171836f8407 100644
--- a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
@@ -363,16 +363,17 @@ SVEFrameSizes AArch64PrologueEpilogueCommon::getSVEStackFrameSizes() const {
StackOffset PPRLocalsSize = AFL.getPPRStackSize(MF) - PPRCalleeSavesSize;
StackOffset ZPRLocalsSize = AFL.getZPRStackSize(MF) - ZPRCalleeSavesSize;
if (AFI->hasSplitSVEObjects())
- return {PPRCalleeSavesSize, ZPRCalleeSavesSize, PPRLocalsSize,
- ZPRLocalsSize};
+ return {{PPRCalleeSavesSize, PPRLocalsSize},
+ {ZPRCalleeSavesSize, ZPRLocalsSize}};
// For simplicity, attribute all locals to ZPRs when split SVE is disabled.
- return {PPRCalleeSavesSize, ZPRCalleeSavesSize, StackOffset{},
- PPRLocalsSize + ZPRLocalsSize};
+ return {{PPRCalleeSavesSize, StackOffset{}},
+ {ZPRCalleeSavesSize, PPRLocalsSize + ZPRLocalsSize}};
}
struct SVEPartitions {
- MachineBasicBlock::iterator PPRBegin, PPREnd;
- MachineBasicBlock::iterator ZPRBegin, ZPREnd;
+ struct {
+ MachineBasicBlock::iterator Begin, End;
+ } PPR, ZPR;
};
static SVEPartitions partitionSVECS(MachineBasicBlock &MBB,
@@ -399,8 +400,8 @@ static SVEPartitions partitionSVECS(MachineBasicBlock &MBB,
IsEpilogue ? (--ZPRsI) : (++ZPRsI);
}
if (IsEpilogue)
- return {PPRsI, MBBI, ZPRsI, PPRsI};
- return {MBBI, PPRsI, PPRsI, ZPRsI};
+ return {{PPRsI, MBBI}, {ZPRsI, PPRsI}};
+ return {{MBBI, PPRsI}, {PPRsI, ZPRsI}};
}
AArch64PrologueEmitter::AArch64PrologueEmitter(MachineFunction &MF,
@@ -763,61 +764,59 @@ void AArch64PrologueEmitter::emitPrologue() {
if (AFL.windowsRequiresStackProbe(MF, NumBytes + RealignmentPadding))
emitWindowsStackProbe(AfterGPRSavesI, DL, NumBytes, RealignmentPadding);
- assert(!(AFL.canUseRedZone(MF) && NeedsRealignment) &&
- "Cannot use redzone with stack realignment");
-
- auto [PPRCalleeSavesSize, ZPRCalleeSavesSize, PPRLocalsSize, ZPRLocalsSize] =
- getSVEStackFrameSizes();
+ auto [PPR, ZPR] = getSVEStackFrameSizes();
StackOffset NonSVELocalsSize = StackOffset::getFixed(NumBytes);
- StackOffset SVECalleeSavesSize = ZPRCalleeSavesSize + PPRCalleeSavesSize;
+ StackOffset SVECalleeSavesSize = ZPR.CalleeSavesSize + PPR.CalleeSavesSize;
StackOffset CFAOffset =
StackOffset::getFixed(MFI.getStackSize()) - NonSVELocalsSize;
MachineBasicBlock::iterator AfterSVESavesI = AfterGPRSavesI;
// Allocate space for the callee saves and PPR locals (if any).
if (SVELayout != SVEStackLayout::CalleeSavesAboveFrameRecord) {
- SVEPartitions SVECSPartitions =
- partitionSVECS(MBB, AfterGPRSavesI, PPRCalleeSavesSize,
- ZPRCalleeSavesSize, /*IsEpilogue=*/false);
- AfterSVESavesI = SVECSPartitions.ZPREnd;
+ auto [PPRRange, ZPRRange] =
+ partitionSVECS(MBB, AfterGPRSavesI, PPR.CalleeSavesSize,
+ ZPR.CalleeSavesSize, /*IsEpilogue=*/false);
+ AfterSVESavesI = ZPRRange.End;
if (EmitAsyncCFI)
emitCalleeSavedSVELocations(AfterSVESavesI);
StackOffset AllocateBeforePPRs = SVECalleeSavesSize;
- StackOffset AllocateAfterPPRs = PPRLocalsSize;
+ StackOffset AllocateAfterPPRs = PPR.LocalsSize;
if (SVELayout == SVEStackLayout::Split) {
- AllocateBeforePPRs = PPRCalleeSavesSize;
- AllocateAfterPPRs = PPRLocalsSize + ZPRCalleeSavesSize;
+ AllocateBeforePPRs = PPR.CalleeSavesSize;
+ AllocateAfterPPRs = PPR.LocalsSize + ZPR.CalleeSavesSize;
}
- allocateStackSpace(SVECSPartitions.PPRBegin, 0, AllocateBeforePPRs,
+ allocateStackSpace(PPRRange.Begin, 0, AllocateBeforePPRs,
EmitAsyncCFI && !HasFP, CFAOffset,
MFI.hasVarSizedObjects() || AllocateAfterPPRs ||
- ZPRLocalsSize || NonSVELocalsSize);
+ ZPR.LocalsSize || NonSVELocalsSize);
CFAOffset += AllocateBeforePPRs;
- assert(SVECSPartitions.PPREnd == SVECSPartitions.ZPRBegin &&
+ assert(PPRRange.End == ZPRRange.Begin &&
"Expected ZPR callee saves after PPR locals");
- allocateStackSpace(SVECSPartitions.PPREnd, RealignmentPadding,
- AllocateAfterPPRs, EmitAsyncCFI && !HasFP, CFAOffset,
- MFI.hasVarSizedObjects() || ZPRLocalsSize ||
+ allocateStackSpace(PPRRange.End, RealignmentPadding, AllocateAfterPPRs,
+ EmitAsyncCFI && !HasFP, CFAOffset,
+ MFI.hasVarSizedObjects() || ZPR.LocalsSize ||
NonSVELocalsSize);
CFAOffset += AllocateAfterPPRs;
} else {
// Note: With CalleeSavesAboveFrameRecord the SVE CS have already been
// allocated (and separate PPRLocals are not supported).
- assert(!PPRLocalsSize && "Unexpected PPR locals!");
+ assert(!PPR.LocalsSize && "Unexpected PPR locals!");
CFAOffset += SVECalleeSavesSize;
}
// Allocate space for the rest of the frame including ZPR locals. Align the
// stack as necessary.
+ assert(!(AFL.canUseRedZone(MF) && NeedsRealignment) &&
+ "Cannot use redzone with stack realignment");
if (!AFL.canUseRedZone(MF)) {
// FIXME: in the case of dynamic re-alignment, NumBytes doesn't have the
// correct value here, as NumBytes also includes padding bytes, which
// shouldn't be counted here.
- allocateStackSpace(AfterSVESavesI, RealignmentPadding,
- ZPRLocalsSize + NonSVELocalsSize, EmitAsyncCFI && !HasFP,
- CFAOffset, MFI.hasVarSizedObjects());
+ allocateStackSpace(
+ AfterSVESavesI, RealignmentPadding, ZPR.LocalsSize + NonSVELocalsSize,
+ EmitAsyncCFI && !HasFP, CFAOffset, MFI.hasVarSizedObjects());
}
// If we need a base pointer, set it up here. It's whatever the value of the
@@ -1439,24 +1438,24 @@ void AArch64EpilogueEmitter::emitEpilogue() {
NumBytes -= PrologueSaveSize;
assert(NumBytes >= 0 && "Negative stack allocation size!?");
- auto [PPRCalleeSavesSize, ZPRCalleeSavesSize, PPRLocalsSize, ZPRLocalsSize] =
- getSVEStackFrameSizes();
- StackOffset SVECalleeSavesSize = ZPRCalleeSavesSize + PPRCalleeSavesSize;
- StackOffset SVEStackSize = SVECalleeSavesSize + PPRLocalsSize + ZPRLocalsSize;
+ auto [PPR, ZPR] = getSVEStackFrameSizes();
+ StackOffset SVECalleeSavesSize = ZPR.CalleeSavesSize + PPR.CalleeSavesSize;
+ StackOffset SVEStackSize =
+ SVECalleeSavesSize + PPR.LocalsSize + ZPR.LocalsSize;
- auto SVECSPartitions = partitionSVECS(
+ auto [PPRRange, ZPRRange] = partitionSVECS(
MBB,
SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord
? MBB.getFirstTerminator()
: FirstGPRRestoreI,
- PPRCalleeSavesSize, ZPRCalleeSavesSize, /*IsEpilogue=*/true);
+ PPR.CalleeSavesSize, ZPR.CalleeSavesSize, /*IsEpilogue=*/true);
- MachineBasicBlock::iterator RestoreBegin = SVECSPartitions.ZPRBegin;
- MachineBasicBlock::iterator RestoreEnd = SVECSPartitions.PPREnd;
+ MachineBasicBlock::iterator RestoreBegin = ZPRRange.Begin;
+ MachineBasicBlock::iterator RestoreEnd = PPRRange.End;
// Deallocate the SVE area.
if (SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord) {
- StackOffset SVELocalsSize = ZPRLocalsSize + PPRLocalsSize;
+ StackOffset SVELocalsSize = ZPR.LocalsSize + PPR.LocalsSize;
// If the callee-save area is before FP, restoring the FP implicitly
// deallocates non-callee-save SVE allocations. Otherwise, deallocate them
// explicitly.
@@ -1481,8 +1480,6 @@ void AArch64EpilogueEmitter::emitEpilogue() {
SVECalleeSavesSize, TII, MachineInstr::FrameDestroy, false,
NeedsWinCFI, &HasWinCFI);
} else if (AFI->hasSVEStackSize()) {
- auto [PPRRestoreBegin, PPRRestoreEnd, ZPRRestoreBegin, ZPRRestoreEnd] =
- SVECSPartitions;
// If we have stack realignment or variable-sized objects we must use the FP
// to restore SVE callee saves (as there is an unknown amount of
// data/padding between the SP and SVE CS area).
@@ -1520,36 +1517,36 @@ void AArch64EpilogueEmitter::emitEpilogue() {
// Deallocate the non-SVE locals first before we can deallocate (and
// restore callee saves) from the SVE area.
auto NonSVELocals = StackOffset::getFixed(NumBytes);
- emitFrameOffset(MBB, ZPRRestoreBegin, DL, AArch64::SP, AArch64::SP,
+ emitFrameOffset(MBB, ZPRRange.Begin, DL, AArch64::SP, AArch64::SP,
NonSVELocals, TII, MachineInstr::FrameDestroy, false,
NeedsWinCFI, &HasWinCFI, EmitCFI && !HasFP, CFAOffset);
CFAOffset -= NonSVELocals;
NumBytes = 0;
}
- if (ZPRLocalsSize) {
- emitFrameOffset(MBB, ZPRRestoreBegin, DL, AArch64::SP, AArch64::SP,
- ZPRLocalsSize, TII, MachineInstr::FrameDestroy, false,
+ if (ZPR.LocalsSize) {
+ emitFrameOffset(MBB, ZPRRange.Begin, DL, AArch64::SP, AArch64::SP,
+ ZPR.LocalsSize, TII, MachineInstr::FrameDestroy, false,
NeedsWinCFI, &HasWinCFI, EmitCFI && !HasFP, CFAOffset);
- CFAOffset -= ZPRLocalsSize;
+ CFAOffset -= ZPR.LocalsSize;
}
StackOffset SVECalleeSavesToDealloc = SVECalleeSavesSize;
if (SVELayout == SVEStackLayout::Split &&
- (PPRLocalsSize || ZPRCalleeSavesSize)) {
- assert(PPRRestoreBegin == ZPRRestoreEnd &&
+ (PPR.LocalsSize || ZPR.CalleeSavesSize)) {
+ assert(PPRRange.Begin == ZPRRange.End &&
"Expected PPR restores after ZPR");
- emitFrameOffset(MBB, PPRRestoreBegin, DL, AArch64::SP, AArch64::SP,
- PPRLocalsSize + ZPRCalleeSavesSize, TII,
+ emitFrameOffset(MBB, PPRRange.Begin, DL, AArch64::SP, AArch64::SP,
+ PPR.LocalsSize + ZPR.CalleeSavesSize, TII,
MachineInstr::FrameDestroy, false, NeedsWinCFI,
&HasWinCFI, EmitCFI && !HasFP, CFAOffset);
- CFAOffset -= PPRLocalsSize + ZPRCalleeSavesSize;
- SVECalleeSavesToDealloc -= ZPRCalleeSavesSize;
+ CFAOffset -= PPR.LocalsSize + ZPR.CalleeSavesSize;
+ SVECalleeSavesToDealloc -= ZPR.CalleeSavesSize;
}
// If split SVE is on, this dealloc PPRs, otherwise, deallocs ZPRs + PPRs:
if (SVECalleeSavesToDealloc)
- emitFrameOffset(MBB, PPRRestoreEnd, DL, AArch64::SP, AArch64::SP,
+ emitFrameOffset(MBB, PPRRange.End, DL, AArch64::SP, AArch64::SP,
SVECalleeSavesToDealloc, TII,
MachineInstr::FrameDestroy, false, NeedsWinCFI,
&HasWinCFI, EmitCFI && !HasFP, CFAOffset);
@@ -1557,7 +1554,7 @@ void AArch64EpilogueEmitter::emitEpilogue() {
if (EmitCFI)
emitCalleeSavedSVERestores(
- SVELayout == SVEStackLayout::Split ? ZPRRestoreEnd : PPRRestoreEnd);
+ SVELayout == SVEStackLayout::Split ? ZPRRange.End : PPRRange.End);
}
if (!HasFP) {
diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h
index 4347b28600b5e..bccaddaad9eec 100644
--- a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h
+++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.h
@@ -28,8 +28,9 @@ class AArch64FunctionInfo;
class AArch64FrameLowering;
struct SVEFrameSizes {
- StackOffset PPRCalleeSavesSize, ZPRCalleeSavesSize;
- StackOffset PPRLocalsSize, ZPRLocalsSize;
+ struct {
+ StackOffset CalleeSavesSize, LocalsSize;
+ } PPR, ZPR;
};
class AArch64PrologueEpilogueCommon {
>From c2a047c7d98a0853c34c6209309fa4908cf1fde6 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Tue, 7 Oct 2025 10:23:19 +0000
Subject: [PATCH 12/13] Tweaks
---
.../lib/Target/AArch64/AArch64PrologueEpilogue.cpp | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
index db171836f8407..16147bddbbb5f 100644
--- a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
@@ -362,7 +362,7 @@ SVEFrameSizes AArch64PrologueEpilogueCommon::getSVEStackFrameSizes() const {
StackOffset::getScalable(AFI->getZPRCalleeSavedStackSize());
StackOffset PPRLocalsSize = AFL.getPPRStackSize(MF) - PPRCalleeSavesSize;
StackOffset ZPRLocalsSize = AFL.getZPRStackSize(MF) - ZPRCalleeSavesSize;
- if (AFI->hasSplitSVEObjects())
+ if (SVELayout == SVEStackLayout::Split)
return {{PPRCalleeSavesSize, PPRLocalsSize},
{ZPRCalleeSavesSize, ZPRLocalsSize}};
// For simplicity, attribute all locals to ZPRs when split SVE is disabled.
@@ -765,9 +765,8 @@ void AArch64PrologueEmitter::emitPrologue() {
emitWindowsStackProbe(AfterGPRSavesI, DL, NumBytes, RealignmentPadding);
auto [PPR, ZPR] = getSVEStackFrameSizes();
-
- StackOffset NonSVELocalsSize = StackOffset::getFixed(NumBytes);
StackOffset SVECalleeSavesSize = ZPR.CalleeSavesSize + PPR.CalleeSavesSize;
+ StackOffset NonSVELocalsSize = StackOffset::getFixed(NumBytes);
StackOffset CFAOffset =
StackOffset::getFixed(MFI.getStackSize()) - NonSVELocalsSize;
@@ -1439,10 +1438,6 @@ void AArch64EpilogueEmitter::emitEpilogue() {
assert(NumBytes >= 0 && "Negative stack allocation size!?");
auto [PPR, ZPR] = getSVEStackFrameSizes();
- StackOffset SVECalleeSavesSize = ZPR.CalleeSavesSize + PPR.CalleeSavesSize;
- StackOffset SVEStackSize =
- SVECalleeSavesSize + PPR.LocalsSize + ZPR.LocalsSize;
-
auto [PPRRange, ZPRRange] = partitionSVECS(
MBB,
SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord
@@ -1450,6 +1445,9 @@ void AArch64EpilogueEmitter::emitEpilogue() {
: FirstGPRRestoreI,
PPR.CalleeSavesSize, ZPR.CalleeSavesSize, /*IsEpilogue=*/true);
+ StackOffset SVECalleeSavesSize = ZPR.CalleeSavesSize + PPR.CalleeSavesSize;
+ StackOffset SVEStackSize =
+ SVECalleeSavesSize + PPR.LocalsSize + ZPR.LocalsSize;
MachineBasicBlock::iterator RestoreBegin = ZPRRange.Begin;
MachineBasicBlock::iterator RestoreEnd = PPRRange.End;
@@ -1489,7 +1487,7 @@ void AArch64EpilogueEmitter::emitEpilogue() {
if (SVECalleeSavesSize && BaseForSVEDealloc == AArch64::FP) {
// TODO: Support stack realigment and variable-sized objects.
assert(
- !AFI->hasSplitSVEObjects() &&
+ SVELayout != SVEStackLayout::Split &&
"unexpected stack realignment or variable sized objects with split "
"SVE stack objects");
>From d57e83d2a0ab077dea866cc37b848b58f680bdef Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Tue, 7 Oct 2025 15:51:27 +0000
Subject: [PATCH 13/13] Fixups
---
llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h | 4 ++++
llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp | 8 ++++++--
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
index 91e64e69de6d0..bd0a17d743c02 100644
--- a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
@@ -315,6 +315,8 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
}
void setStackSizeSVE(uint64_t ZPR, uint64_t PPR) {
+ assert(isAligned(Align(16), ZPR) && isAligned(Align(16), PPR) &&
+ "expected SVE stack sizes to be aligned to 16-bytes");
StackSizeZPR = ZPR;
StackSizePPR = PPR;
HasCalculatedStackSizeSVE = true;
@@ -425,6 +427,8 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
// Saves the CalleeSavedStackSize for SVE vectors in 'scalable bytes'
void setSVECalleeSavedStackSize(unsigned ZPR, unsigned PPR) {
+ assert(isAligned(Align(16), ZPR) && isAligned(Align(16), PPR) &&
+ "expected SVE callee-save sizes to be aligned to 16-bytes");
ZPRCalleeSavedStackSize = ZPR;
PPRCalleeSavedStackSize = PPR;
HasSVECalleeSavedStackSize = true;
diff --git a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
index 16147bddbbb5f..f110558fcb0d4 100644
--- a/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PrologueEpilogue.cpp
@@ -799,8 +799,10 @@ void AArch64PrologueEmitter::emitPrologue() {
NonSVELocalsSize);
CFAOffset += AllocateAfterPPRs;
} else {
- // Note: With CalleeSavesAboveFrameRecord the SVE CS have already been
- // allocated (and separate PPRLocals are not supported).
+ assert(SVELayout == SVEStackLayout::CalleeSavesAboveFrameRecord);
+ // Note: With CalleeSavesAboveFrameRecord, the SVE CS have already been
+ // allocated (and separate PPR locals are not supported, all SVE locals,
+ // both PPR and ZPR, are within the ZPR locals area).
assert(!PPR.LocalsSize && "Unexpected PPR locals!");
CFAOffset += SVECalleeSavesSize;
}
@@ -1276,6 +1278,8 @@ void AArch64PrologueEmitter::emitCalleeSavedSVELocations(
StackOffset::getScalable(MFI.getObjectOffset(FI)) -
StackOffset::getFixed(AFI->getCalleeSavedStackSize(MFI));
+ // The scalable vectors are below (lower address) the scalable predicates
+ // with split SVE objects, so we must subtract the size of the predicates.
if (SVELayout == SVEStackLayout::Split &&
MFI.getStackID(FI) == TargetStackID::ScalableVector)
Offset -= PPRStackSize;
More information about the llvm-commits
mailing list