[llvm] [AMDGPU] Support partial and empty WWM pools for SGPR spills (PR #213491)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 1 15:25:12 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Shilei Tian (shiltian)
<details>
<summary>Changes</summary>
[AMDGPU] Support partial and empty WWM pools for SGPR spills
SGPR lane spilling currently treats the WWM VGPR pool as all-or-nothing. This can fail compilation when the requested pool cannot be formed, even though scratch spilling or a smaller spillable pool could make progress.
This PR lets ordinary SGPR spills fall back to scratch when the pool is empty and lets WWM register allocation use a nonempty partial pool. It keeps the full-pool requirement for strict WWM/WQM and explicit spill-carrier preallocation.
The no-pool fallback is recorded in SIMachineFunctionInfo so frame lowering can provide enough emergency scavenging slots. The state is also serialized to preserve the behavior across MIR round trips.
---
This PR was assisted by AI but I reviewed all code changes.
---
Patch is 40.13 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/213491.diff
12 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/SIFrameLowering.cpp (+10-4)
- (modified) llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp (+95-49)
- (modified) llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp (+2)
- (modified) llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h (+13)
- (modified) llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp (+6-3)
- (modified) llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.h (+2)
- (modified) llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll (+4-2)
- (modified) llvm/test/CodeGen/AMDGPU/sgpr-spill-vmem-large-frame.mir (+36)
- (modified) llvm/test/CodeGen/AMDGPU/wwm-regalloc-error.ll (+17-3)
- (added) llvm/test/CodeGen/AMDGPU/wwm-regalloc-memory-fallback.ll (+213)
- (added) llvm/test/CodeGen/AMDGPU/wwm-regalloc-partial-pool.mir (+50)
- (added) llvm/test/CodeGen/AMDGPU/wwm-regalloc-preallocation-guard.mir (+101)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
index bcd8d0bef062d..c1fa828f38bbf 100644
--- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -1793,10 +1793,16 @@ void SIFrameLowering::processFunctionBeforeFrameFinalized(
// Add an emergency spill slot
RS->addScavengingFrameIndex(FuncInfo->getScavengeFI(MFI, *TRI));
- // If we are spilling SGPRs to memory with a large frame, we may need a
- // second VGPR emergency frame index.
- if (HaveSGPRToVMemSpill &&
- allocateScavengingFrameIndexesNearIncomingSP(MF)) {
+ if (HaveSGPRToVMemSpill && FuncInfo->hasNoWWMPoolSGPRSpillFallback()) {
+ // The no-WWM-pool fallback can reach SGPR-to-memory lowering while an
+ // ordinary frame-index scavenge is live. It may then need one slot for
+ // its temporary VGPR and another for recursive address materialization.
+ RS->addScavengingFrameIndex(MFI.CreateSpillStackObject(4, Align(4)));
+ RS->addScavengingFrameIndex(MFI.CreateSpillStackObject(4, Align(4)));
+ } else if (HaveSGPRToVMemSpill &&
+ allocateScavengingFrameIndexesNearIncomingSP(MF)) {
+ // Existing large-frame SGPR-to-memory spills need one additional VGPR
+ // emergency frame index.
RS->addScavengingFrameIndex(MFI.CreateSpillStackObject(4, Align(4)));
}
}
diff --git a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
index 8cc7714c6fa2e..5bf34ddcb2816 100644
--- a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
@@ -20,6 +20,7 @@
#include "GCNSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIMachineFunctionInfo.h"
+#include "SIPreAllocateWWMRegs.h"
#include "SISpillUtils.h"
#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/MachineCycleAnalysis.h"
@@ -80,7 +81,9 @@ class SILowerSGPRSpills {
void updateLaneVGPRDomInstr(
int FI, MachineBasicBlock *MBB, MachineBasicBlock::iterator InsertPt,
DenseMap<Register, LaneVGPRInsertPt> &LaneVGPRDomInstr);
- void determineRegsForWWMAllocation(MachineFunction &MF, BitVector &RegMask);
+ SmallVector<MCRegister> determineRegsForWWMAllocation(MachineFunction &MF);
+ void assignWWMRegs(MachineFunction &MF, ArrayRef<MCRegister> WWMRegCandidates,
+ bool RequiresFullWWMPool);
};
class SILowerSGPRSpillsLegacy : public MachineFunctionPass {
@@ -372,42 +375,64 @@ void SILowerSGPRSpills::updateLaneVGPRDomInstr(
}
}
-void SILowerSGPRSpills::determineRegsForWWMAllocation(MachineFunction &MF,
- BitVector &RegMask) {
- // Determine an optimal number of VGPRs for WWM allocation. The complement
- // list will be available for allocating other VGPR virtual registers.
- SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+SmallVector<MCRegister>
+SILowerSGPRSpills::determineRegsForWWMAllocation(MachineFunction &MF) {
+ SmallVector<MCRegister> WWMRegCandidates;
+ if (!MaxNumVGPRsForWwmAllocation)
+ return WWMRegCandidates;
+
MachineRegisterInfo &MRI = MF.getRegInfo();
BitVector ReservedRegs = TRI->getReservedRegs(MF);
- BitVector NonWwmAllocMask(TRI->getNumRegs());
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ unsigned MaxNumVGPRs = ST.getMaxNumVectorRegs(MF.getFunction()).first;
- // FIXME: MaxNumVGPRsForWwmAllocation might need to be adjusted in the future
- // to have a balanced allocation between WWM values and per-thread vector
- // register operands.
- unsigned NumRegs = MaxNumVGPRsForWwmAllocation;
- NumRegs =
- std::min(static_cast<unsigned>(MFI->getSGPRSpillVGPRs().size()), NumRegs);
-
- auto [MaxNumVGPRs, MaxNumAGPRs] = ST.getMaxNumVectorRegs(MF.getFunction());
// Try to use the highest available registers for now. Later after
// vgpr-regalloc, they can be shifted to the lowest range.
- unsigned I = 0;
for (unsigned Reg = AMDGPU::VGPR0 + MaxNumVGPRs - 1;
- (I < NumRegs) && (Reg >= AMDGPU::VGPR0); --Reg) {
+ WWMRegCandidates.size() < MaxNumVGPRsForWwmAllocation &&
+ Reg >= AMDGPU::VGPR0;
+ --Reg) {
if (!ReservedRegs.test(Reg) &&
- !MRI.isPhysRegUsed(Reg, /*SkipRegMaskTest=*/true)) {
- TRI->markSuperRegs(RegMask, Reg);
- ++I;
- }
+ !MRI.isPhysRegUsed(Reg, /*SkipRegMaskTest=*/true))
+ WWMRegCandidates.push_back(Reg);
}
- if (I != NumRegs) {
+ return WWMRegCandidates;
+}
+
+void SILowerSGPRSpills::assignWWMRegs(MachineFunction &MF,
+ ArrayRef<MCRegister> WWMRegCandidates,
+ bool RequiresFullWWMPool) {
+ SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
+ if (FuncInfo->getSGPRSpillVGPRs().empty())
+ return;
+
+ BitVector WwmRegMask(TRI->getNumRegs());
+
+ unsigned DesiredPoolSize =
+ std::min(static_cast<unsigned>(FuncInfo->getSGPRSpillVGPRs().size()),
+ static_cast<unsigned>(MaxNumVGPRsForWwmAllocation));
+ unsigned SelectedPoolSize =
+ std::min<unsigned>(DesiredPoolSize, WWMRegCandidates.size());
+ // WWM register candidates are ordered high-to-low, so take the highest
+ // available registers when the desired pool is smaller than the candidate
+ // list.
+ for (MCRegister Reg : WWMRegCandidates.take_front(SelectedPoolSize))
+ TRI->markSuperRegs(WwmRegMask, Reg);
+
+ if (RequiresFullWWMPool && SelectedPoolSize != DesiredPoolSize) {
// Reserve an arbitrary register and report the error.
- TRI->markSuperRegs(RegMask, AMDGPU::VGPR0);
+ TRI->markSuperRegs(WwmRegMask, AMDGPU::VGPR0);
MF.getFunction().getContext().emitError(
"cannot find enough VGPRs for wwm-regalloc");
}
+
+ BitVector NonWwmRegMask(WwmRegMask);
+ NonWwmRegMask.flip().clearBitsNotInMask(TRI->getAllVGPRRegMask());
+
+ // The complement set will be the registers for non-wwm (per-thread) vgpr
+ // allocation.
+ FuncInfo->updateNonWWMRegMask(NonWwmRegMask);
}
bool SILowerSGPRSpillsLegacy::runOnMachineFunction(MachineFunction &MF) {
@@ -465,8 +490,19 @@ bool SILowerSGPRSpills::run(MachineFunction &MF) {
// To track the IMPLICIT_DEF insertion point for the lane vgprs.
DenseMap<Register, LaneVGPRInsertPt> LaneVGPRDomInstr;
+ // Defer ordinary spills until physical CSR spills have reserved their
+ // lane VGPRs and the WWM allocation pool can be selected.
+ SmallVector<MachineInstr *> OrdinarySGPRSpills;
+ bool HasStrictWWMRegion = false;
+
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : llvm::make_early_inc_range(MBB)) {
+ if (MI.getOpcode() == AMDGPU::ENTER_STRICT_WWM ||
+ MI.getOpcode() == AMDGPU::ENTER_STRICT_WQM) {
+ HasStrictWWMRegion = true;
+ continue;
+ }
+
if (!TII->isSGPRSpill(MI))
continue;
@@ -500,18 +536,39 @@ bool SILowerSGPRSpills::run(MachineFunction &MF) {
llvm_unreachable(
"failed to spill SGPR to physical VGPR lane when allocated");
}
- } else {
- MachineInstrSpan MIS(&MI, &MBB);
- if (FuncInfo->allocateSGPRSpillToVGPRLane(MF, FI)) {
- bool Spilled = TRI->eliminateSGPRToVGPRSpillFrameIndex(
- MI, FI, nullptr, Indexes, LIS);
- if (!Spilled)
- llvm_unreachable(
- "failed to spill SGPR to virtual VGPR lane when allocated");
- SpillFIs.set(FI);
- updateLaneVGPRDomInstr(FI, &MBB, MIS.begin(), LaneVGPRDomInstr);
- SpilledToVirtVGPRLanes = true;
- }
+ } else
+ OrdinarySGPRSpills.push_back(&MI);
+ }
+ }
+
+ // Select candidates once, before ordinary lane lowering creates virtual
+ // VGPRs and changes the number of registers desired for the WWM pool.
+ SmallVector<MCRegister> WWMRegCandidates;
+ // These non-spillable WWM users retain the old all-or-nothing pool policy.
+ const bool RequiresFullWWMPool =
+ HasStrictWWMRegion || isPreallocateSGPRSpillVGPRsEnabled(MF);
+ if (!OrdinarySGPRSpills.empty())
+ WWMRegCandidates = determineRegsForWWMAllocation(MF);
+
+ const bool ShouldLowerOrdinarySpillsToVGPRLanes =
+ RequiresFullWWMPool || !WWMRegCandidates.empty();
+ if (!ShouldLowerOrdinarySpillsToVGPRLanes && !OrdinarySGPRSpills.empty())
+ FuncInfo->setNoWWMPoolSGPRSpillFallback();
+
+ if (ShouldLowerOrdinarySpillsToVGPRLanes) {
+ for (MachineInstr *MI : OrdinarySGPRSpills) {
+ int FI = TII->getNamedOperand(*MI, AMDGPU::OpName::addr)->getIndex();
+ if (FuncInfo->allocateSGPRSpillToVGPRLane(MF, FI)) {
+ MachineBasicBlock *MBB = MI->getParent();
+ MachineInstrSpan MIS(MI, MBB);
+ bool Spilled = TRI->eliminateSGPRToVGPRSpillFrameIndex(
+ *MI, FI, nullptr, Indexes, LIS);
+ if (!Spilled)
+ llvm_unreachable(
+ "failed to spill SGPR to virtual VGPR lane when allocated");
+ SpillFIs.set(FI);
+ updateLaneVGPRDomInstr(FI, MBB, MIS.begin(), LaneVGPRDomInstr);
+ SpilledToVirtVGPRLanes = true;
}
}
}
@@ -538,20 +595,9 @@ bool SILowerSGPRSpills::run(MachineFunction &MF) {
}
}
- // Determine the registers for WWM allocation and also compute the register
- // mask for non-wwm VGPR allocation.
- if (FuncInfo->getSGPRSpillVGPRs().size()) {
- BitVector WwmRegMask(TRI->getNumRegs());
-
- determineRegsForWWMAllocation(MF, WwmRegMask);
-
- BitVector NonWwmRegMask(WwmRegMask);
- NonWwmRegMask.flip().clearBitsNotInMask(TRI->getAllVGPRRegMask());
-
- // The complement set will be the registers for non-wwm (per-thread) vgpr
- // allocation.
- FuncInfo->updateNonWWMRegMask(NonWwmRegMask);
- }
+ // Assign the WWM pool from the pre-selected candidates and compute the
+ // complement mask for per-thread VGPR allocation.
+ assignWWMRegs(MF, WWMRegCandidates, RequiresFullWWMPool);
for (MachineBasicBlock &MBB : MF)
clearDebugInfoForSpillFIs(MFI, MBB, SpillFIs);
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
index 59971923e4a5d..950019b37a462 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
@@ -739,6 +739,7 @@ yaml::SIMachineFunctionInfo::SIMachineFunctionInfo(
WaveLimiter(MFI.needsWaveLimiter()),
HasSpilledSGPRs(MFI.hasSpilledSGPRs()),
HasSpilledVGPRs(MFI.hasSpilledVGPRs()),
+ HasNoWWMPoolSGPRSpillFallback(MFI.hasNoWWMPoolSGPRSpillFallback()),
NumWaveDispatchSGPRs(MFI.getNumWaveDispatchSGPRs()),
NumWaveDispatchVGPRs(MFI.getNumWaveDispatchVGPRs()),
HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()),
@@ -798,6 +799,7 @@ bool SIMachineFunctionInfo::initializeBaseYamlFields(
WaveLimiter = YamlMFI.WaveLimiter;
HasSpilledSGPRs = YamlMFI.HasSpilledSGPRs;
HasSpilledVGPRs = YamlMFI.HasSpilledVGPRs;
+ HasNoWWMPoolSGPRSpillFallback = YamlMFI.HasNoWWMPoolSGPRSpillFallback;
NumWaveDispatchSGPRs = YamlMFI.NumWaveDispatchSGPRs;
NumWaveDispatchVGPRs = YamlMFI.NumWaveDispatchVGPRs;
BytesInStackArgArea = YamlMFI.BytesInStackArgArea;
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
index 7374e996837a5..2d1b77fc4fb8f 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
@@ -272,6 +272,7 @@ struct SIMachineFunctionInfo final : public yaml::MachineFunctionInfo {
bool WaveLimiter = false;
bool HasSpilledSGPRs = false;
bool HasSpilledVGPRs = false;
+ bool HasNoWWMPoolSGPRSpillFallback = false;
uint16_t NumWaveDispatchSGPRs = 0;
uint16_t NumWaveDispatchVGPRs = 0;
uint32_t HighBitsOf32BitAddress = 0;
@@ -334,6 +335,8 @@ template <> struct MappingTraits<SIMachineFunctionInfo> {
YamlIO.mapOptional("waveLimiter", MFI.WaveLimiter, false);
YamlIO.mapOptional("hasSpilledSGPRs", MFI.HasSpilledSGPRs, false);
YamlIO.mapOptional("hasSpilledVGPRs", MFI.HasSpilledVGPRs, false);
+ YamlIO.mapOptional("hasNoWWMPoolSGPRSpillFallback",
+ MFI.HasNoWWMPoolSGPRSpillFallback, false);
YamlIO.mapOptional("numWaveDispatchSGPRs", MFI.NumWaveDispatchSGPRs, false);
YamlIO.mapOptional("numWaveDispatchVGPRs", MFI.NumWaveDispatchVGPRs, false);
YamlIO.mapOptional("scratchRSrcReg", MFI.ScratchRSrcReg,
@@ -613,6 +616,11 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunctionInfo,
// frame, so save it here and add it to the RegScavenger later.
std::optional<int> ScavengeFI;
+ // Ordinary SGPR spills fell back to memory because no WWM VGPR pool was
+ // available. This path may require additional nested VGPR scavenging slots
+ // during frame-index elimination.
+ bool HasNoWWMPoolSGPRSpillFallback = false;
+
// Map each VGPR CSR to the mask needed to save and restore it using block
// load/store instructions. Only used if the subtarget feature for VGPR block
// load/store is enabled.
@@ -853,6 +861,11 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunctionInfo,
int getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI);
std::optional<int> getOptionalScavengeFI() const { return ScavengeFI; }
+ void setNoWWMPoolSGPRSpillFallback() { HasNoWWMPoolSGPRSpillFallback = true; }
+ bool hasNoWWMPoolSGPRSpillFallback() const {
+ return HasNoWWMPoolSGPRSpillFallback;
+ }
+
unsigned getBytesInStackArgArea() const {
return BytesInStackArgArea;
}
diff --git a/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp b/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp
index bf484cef98da4..0d9ef77b70562 100644
--- a/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp
@@ -35,6 +35,11 @@ static cl::opt<bool>
EnablePreallocateSGPRSpillVGPRs("amdgpu-prealloc-sgpr-spill-vgprs",
cl::init(false), cl::Hidden);
+bool llvm::isPreallocateSGPRSpillVGPRsEnabled(const MachineFunction &MF) {
+ return EnablePreallocateSGPRSpillVGPRs ||
+ MF.getFunction().hasFnAttribute("amdgpu-prealloc-sgpr-spill-vgprs");
+}
+
namespace {
class SIPreAllocateWWMRegs {
@@ -221,9 +226,7 @@ bool SIPreAllocateWWMRegs::run(MachineFunction &MF) {
TRI = &TII->getRegisterInfo();
MRI = &MF.getRegInfo();
- bool PreallocateSGPRSpillVGPRs =
- EnablePreallocateSGPRSpillVGPRs ||
- MF.getFunction().hasFnAttribute("amdgpu-prealloc-sgpr-spill-vgprs");
+ bool PreallocateSGPRSpillVGPRs = isPreallocateSGPRSpillVGPRsEnabled(MF);
bool RegsAssigned = false;
diff --git a/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.h b/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.h
index 8d3fa21035966..2fcd9f9cba351 100644
--- a/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.h
+++ b/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.h
@@ -13,6 +13,8 @@
namespace llvm {
+bool isPreallocateSGPRSpillVGPRsEnabled(const MachineFunction &MF);
+
class SIPreAllocateWWMRegsPass
: public RequiredPassInfoMixin<SIPreAllocateWWMRegsPass> {
public:
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
index e9bb77f3db5dd..30202943afa84 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-amdgpu-tracker-physreg-crash.ll
@@ -16,7 +16,10 @@
i64 ; vcc
}
-; ERR-GCNTRACKERS: ran out of registers during register allocation
+; With the tracker enabled, no separate WWM VGPR is available and the SGPR
+; spill falls back to memory. This case cannot preserve EXEC because SCC is
+; live and no SGPR can be scavenged.
+; ERR-GCNTRACKERS: unhandled SGPR spill to memory
; GCN-NOT: ran out of registers during register allocation
; FIXME: GCN Trackers do not track pressure from PhysRegs, so scheduling is actually worse
@@ -62,4 +65,3 @@ define void @scalar_mov_materializes_frame_index_no_live_scc_no_live_sgprs() #0
attributes #0 = { nounwind alignstack=64 "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="10,10" "no-realign-stack" }
attributes #1 = { nounwind alignstack=16 "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="10,10" "no-realign-stack" }
-
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-vmem-large-frame.mir b/llvm/test/CodeGen/AMDGPU/sgpr-spill-vmem-large-frame.mir
index b23aa0f45a05c..ccb963d7fa8ab 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-vmem-large-frame.mir
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-vmem-large-frame.mir
@@ -4,6 +4,10 @@
# Check that we allocate 2 emergency stack slots if we're spilling
# SGPRs to memory and potentially have an offset larger than fits in
# the addressing mode of the memory instructions.
+#
+# The no-WWM-pool fallback needs 3 emergency slots even for a small frame:
+# one for an ordinary frame-index scavenge, one for the SGPR spill temporary,
+# and one for recursive address materialization.
---
name: test
@@ -49,3 +53,35 @@ body: |
renamable $sgpr10 = SI_SPILL_S32_RESTORE %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
S_SETPC_B64 $sgpr30_sgpr31, implicit $scc
...
+---
+name: fallback_scavenging
+tracksRegLiveness: true
+frameInfo:
+ maxAlignment: 4
+stack:
+ - { id: 0, type: spill-slot, size: 4, alignment: 4, stack-id: sgpr-spill }
+machineFunctionInfo:
+ isEntryFunction: false
+ scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3'
+ stackPtrOffsetReg: '$sgpr32'
+ frameOffsetReg: '$sgpr33'
+ hasSpilledSGPRs: true
+ hasNoWWMPoolSGPRSpillFallback: true
+body: |
+ bb.0:
+ liveins: $sgpr30_sgpr31, $sgpr10, $sgpr11
+ ; CHECK-LABEL: name: fallback_scavenging
+ ; CHECK: frameInfo:
+ ; CHECK: stackSize: 16
+ ; CHECK: stack:
+ ; CHECK: - { id: 0,
+ ; CHECK: - { id: 1,
+ ; CHECK: - { id: 2,
+ ; CHECK: - { id: 3,
+ ; CHECK: machineFunctionInfo:
+ ; CHECK: hasNoWWMPoolSGPRSpillFallback: true
+ S_CMP_EQ_U32 0, 0, implicit-def $scc
+ SI_SPILL_S32_SAVE killed $sgpr10, %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+ renamable $sgpr10 = SI_SPILL_S32_RESTORE %stack.0, implicit $exec, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr32
+ S_SETPC_B64 $sgpr30_sgpr31, implicit $scc
+...
diff --git a/llvm/test/CodeGen/AMDGPU/wwm-regalloc-error.ll b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-error.ll
index 2367af90d9555..557c04a9ae38c 100644
--- a/llvm/test/CodeGen/AMDGPU/wwm-regalloc-error.ll
+++ b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-error.ll
@@ -1,8 +1,22 @@
-; RUN: not llc -mtriple=amdgpu9.00-amd-amdhsa -stress-regalloc=2 -filetype=null %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -stress-regalloc=2 -o - %s | FileCheck %s
+; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -stress-regalloc=2 -stop-after=si-lower-sgpr-spills -o %t.mir %s
+; RUN: FileCheck --check-prefix=MIR %s < %t.mir
+; RUN: llc -mtriple=amdgpu9.00-amd-amdhsa -stress-regalloc=2 -start-after=si-lower-sgpr-spills -o - %t.mir | FileCheck --check-prefix=ROUNDTRIP %s
-; A negative test to capture the expected error when the VGPRs are insufficient for wwm-regalloc.
+; All allocatable VGPRs are mentioned by inline assembly. Ordinary SGPR spills
+; must fall back to scratch memory when no separate WWM pool is available.
-; CHECK: error: cannot find enough VGPRs for wwm-regalloc
+; CHECK-LABEL: test:
+; CHECK: buffer_store_dword {{.*}} ; 4-byte Folded Spill
+; CHECK: buffer_load_dword {{.*}} ; 4-byte Folded Reload
+; CHECK: s_endpgm
+; CHECK: .amdhsa_private_segment_fixed_size 20
+
+; MIR: hasNoWWMPoolSGPRSpillFallback: true
+
+; ROUNDTRIP-LABEL: test:
+; ROUNDTRIP: s_endpgm
+; ROUNDTRIP: .amdhsa_private_segment_fixed_size 20
define amdgpu_kernel void @test(i32 %in) {
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/wwm-regalloc-memory-fallback.ll b/llvm/test/CodeGen/AMDGPU/wwm-regalloc-memory-fallback.ll
new file mode 100644
index 0000000000000..fa30f46dc83c7
--- /dev/null
+++ b/llvm/test/C...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/213491
More information about the llvm-commits
mailing list