[llvm] 117e560 - [AMDGPU] Reserving VGPR for future SGPR Spill
Saiyedul Islam via llvm-commits
llvm-commits at lists.llvm.org
Mon May 11 17:33:19 PDT 2020
Author: Saiyedul Islam
Date: 2020-05-12T00:33:00Z
New Revision: 117e5609e98b43f925c678b72f816ad3a1c3eee7
URL: https://github.com/llvm/llvm-project/commit/117e5609e98b43f925c678b72f816ad3a1c3eee7
DIFF: https://github.com/llvm/llvm-project/commit/117e5609e98b43f925c678b72f816ad3a1c3eee7.diff
LOG: [AMDGPU] Reserving VGPR for future SGPR Spill
Summary: One VGPR register is allocated to handle a future spill of SGPR if "--amdgpu-reserve-vgpr-for-sgpr-spill" option is used
Reviewers: arsenm, rampitec, msearles, cdevadas
Reviewed By: arsenm
Subscribers: madhur13490, qcolombet, kerbowa, kzhuravl, jvesely, wdng, nhaehnle, yaxunl, dstuttard, tpr, t-tye, hiraditya, llvm-commits
Tags: #amdgpu, #llvm
Differential Revision: https://reviews.llvm.org/D70379
Added:
llvm/test/CodeGen/AMDGPU/reserve-vgpr-for-sgpr-spill.ll
Modified:
llvm/lib/Target/AMDGPU/SIISelLowering.cpp
llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
llvm/lib/Target/AMDGPU/SIRegisterInfo.h
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index a2abc45a38e5..9f155125266b 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -95,6 +95,10 @@ static cl::opt<bool> DisableLoopAlignment(
cl::desc("Do not align and prefetch loops"),
cl::init(false));
+static cl::opt<bool> VGPRReserveforSGPRSpill(
+ "amdgpu-reserve-vgpr-for-sgpr-spill",
+ cl::desc("Allocates one VGPR for future SGPR Spill"), cl::init(true));
+
static bool hasFP32Denormals(const MachineFunction &MF) {
const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
return Info->getMode().allFP32Denormals();
@@ -10858,6 +10862,13 @@ void SITargetLowering::finalizeLowering(MachineFunction &MF) const {
}
TargetLoweringBase::finalizeLowering(MF);
+
+ // Allocate a VGPR for future SGPR Spill if
+ // "amdgpu-reserve-vgpr-for-sgpr-spill" option is used
+ // FIXME: We won't need this hack if we split SGPR allocation from VGPR
+ if (VGPRReserveforSGPRSpill && !Info->VGPRReservedForSGPRSpill &&
+ !Info->isEntryFunction() && MF.getFrameInfo().hasStackObjects())
+ Info->reserveVGPRforSGPRSpills(MF);
}
void SITargetLowering::computeKnownBitsForFrameIndex(const SDValue Op,
diff --git a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
index 2cf3054fd455..7631a0e72d8b 100644
--- a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
@@ -231,6 +231,52 @@ bool SILowerSGPRSpills::spillCalleeSavedRegs(MachineFunction &MF) {
return false;
}
+static ArrayRef<MCPhysReg> getAllVGPR32(const GCNSubtarget &ST,
+ const MachineFunction &MF) {
+ return makeArrayRef(AMDGPU::VGPR_32RegClass.begin(), ST.getMaxNumVGPRs(MF));
+}
+
+// Find lowest available VGPR and use it as VGPR reserved for SGPR spills.
+static bool lowerShiftReservedVGPR(MachineFunction &MF,
+ const GCNSubtarget &ST) {
+ MachineRegisterInfo &MRI = MF.getRegInfo();
+ MachineFrameInfo &FrameInfo = MF.getFrameInfo();
+ SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
+ Register LowestAvailableVGPR, ReservedVGPR;
+ ArrayRef<MCPhysReg> AllVGPR32s = getAllVGPR32(ST, MF);
+ for (MCPhysReg Reg : AllVGPR32s) {
+ if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg)) {
+ LowestAvailableVGPR = Reg;
+ break;
+ }
+ }
+
+ if (!LowestAvailableVGPR)
+ return false;
+
+ ReservedVGPR = FuncInfo->VGPRReservedForSGPRSpill;
+ const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
+ int i = 0;
+
+ for (MachineBasicBlock &MBB : MF) {
+ for (auto Reg : FuncInfo->getSGPRSpillVGPRs()) {
+ if (Reg.VGPR == ReservedVGPR) {
+ MBB.removeLiveIn(ReservedVGPR);
+ MBB.addLiveIn(LowestAvailableVGPR);
+ Optional<int> FI;
+ if (FuncInfo->isCalleeSavedReg(CSRegs, LowestAvailableVGPR))
+ FI = FrameInfo.CreateSpillStackObject(4, Align(4));
+
+ FuncInfo->setSGPRSpillVGPRs(LowestAvailableVGPR, FI, i);
+ }
+ ++i;
+ }
+ MBB.sortUniqueLiveIns();
+ }
+
+ return true;
+}
+
bool SILowerSGPRSpills::runOnMachineFunction(MachineFunction &MF) {
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
TII = ST.getInstrInfo();
@@ -270,6 +316,9 @@ bool SILowerSGPRSpills::runOnMachineFunction(MachineFunction &MF) {
//
// This operates under the assumption that only other SGPR spills are users
// of the frame index.
+
+ lowerShiftReservedVGPR(MF, ST);
+
for (MachineBasicBlock &MBB : MF) {
MachineBasicBlock::iterator Next;
for (auto I = MBB.begin(), E = MBB.end(); I != E; I = Next) {
@@ -318,6 +367,8 @@ bool SILowerSGPRSpills::runOnMachineFunction(MachineFunction &MF) {
}
MadeChange = true;
+ } else if (FuncInfo->VGPRReservedForSGPRSpill) {
+ FuncInfo->removeVGPRForSGPRSpill(FuncInfo->VGPRReservedForSGPRSpill, MF);
}
SaveBlocks.clear();
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
index 104138ea0f05..8f25ebd3437e 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
@@ -261,7 +261,8 @@ Register SIMachineFunctionInfo::addImplicitBufferPtr(const SIRegisterInfo &TRI)
return ArgInfo.ImplicitBufferPtr.getRegister();
}
-static bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg) {
+bool SIMachineFunctionInfo::isCalleeSavedReg(const MCPhysReg *CSRegs,
+ MCPhysReg Reg) {
for (unsigned I = 0; CSRegs[I]; ++I) {
if (CSRegs[I] == Reg)
return true;
@@ -295,6 +296,7 @@ bool SIMachineFunctionInfo::allocateSGPRSpillToVGPR(MachineFunction &MF,
MachineFrameInfo &FrameInfo = MF.getFrameInfo();
MachineRegisterInfo &MRI = MF.getRegInfo();
unsigned WaveSize = ST.getWavefrontSize();
+ SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
unsigned Size = FrameInfo.getObjectSize(FI);
assert(Size >= 4 && Size <= 64 && "invalid sgpr spill size");
@@ -310,7 +312,7 @@ bool SIMachineFunctionInfo::allocateSGPRSpillToVGPR(MachineFunction &MF,
Register LaneVGPR;
unsigned VGPRIndex = (NumVGPRSpillLanes % WaveSize);
- if (VGPRIndex == 0) {
+ if (VGPRIndex == 0 && !FuncInfo->VGPRReservedForSGPRSpill) {
LaneVGPR = TRI->findUnusedRegister(MRI, &AMDGPU::VGPR_32RegClass, MF);
if (LaneVGPR == AMDGPU::NoRegister) {
// We have no VGPRs left for spilling SGPRs. Reset because we will not
@@ -342,6 +344,19 @@ bool SIMachineFunctionInfo::allocateSGPRSpillToVGPR(MachineFunction &MF,
return true;
}
+/// Reserve a VGPR for spilling of SGPRs
+bool SIMachineFunctionInfo::reserveVGPRforSGPRSpills(MachineFunction &MF) {
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ const SIRegisterInfo *TRI = ST.getRegisterInfo();
+ SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
+
+ Register LaneVGPR = TRI->findUnusedRegister(
+ MF.getRegInfo(), &AMDGPU::VGPR_32RegClass, MF, true);
+ SpillVGPRs.push_back(SGPRSpillVGPRCSR(LaneVGPR, None));
+ FuncInfo->VGPRReservedForSGPRSpill = LaneVGPR;
+ return true;
+}
+
/// Reserve AGPRs or VGPRs to support spilling for FrameIndex \p FI.
/// Either AGPR is spilled to VGPR to vice versa.
/// Returns true if a \p FI can be eliminated completely.
@@ -554,3 +569,21 @@ bool SIMachineFunctionInfo::initializeBaseYamlFields(
WaveLimiter = YamlMFI.WaveLimiter;
return false;
}
+
+// Remove VGPR which was reserved for SGPR spills if there are no spilled SGPRs
+bool SIMachineFunctionInfo::removeVGPRForSGPRSpill(Register ReservedVGPR,
+ MachineFunction &MF) {
+ for (auto *i = SpillVGPRs.begin(); i < SpillVGPRs.end(); i++) {
+ if (i->VGPR == ReservedVGPR) {
+ SpillVGPRs.erase(i);
+
+ for (MachineBasicBlock &MBB : MF) {
+ MBB.removeLiveIn(ReservedVGPR);
+ MBB.sortUniqueLiveIns();
+ }
+ this->VGPRReservedForSGPRSpill = AMDGPU::NoRegister;
+ return true;
+ }
+ }
+ return false;
+}
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
index ae04896bef72..0d85136e23e0 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
@@ -485,6 +485,9 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunction {
Register SGPRForFPSaveRestoreCopy;
Optional<int> FramePointerSaveIndex;
+ Register VGPRReservedForSGPRSpill;
+ bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg);
+
public:
SIMachineFunctionInfo(const MachineFunction &MF);
@@ -500,6 +503,14 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunction {
return SpillVGPRs;
}
+ void setSGPRSpillVGPRs(Register NewVGPR, Optional<int> newFI, int Index) {
+ SpillVGPRs[Index].VGPR = NewVGPR;
+ SpillVGPRs[Index].FI = newFI;
+ VGPRReservedForSGPRSpill = NewVGPR;
+ }
+
+ bool removeVGPRForSGPRSpill(Register ReservedVGPR, MachineFunction &MF);
+
ArrayRef<MCPhysReg> getAGPRSpillVGPRs() const {
return SpillAGPR;
}
@@ -517,6 +528,7 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunction {
bool haveFreeLanesForSGPRSpill(const MachineFunction &MF,
unsigned NumLane) const;
bool allocateSGPRSpillToVGPR(MachineFunction &MF, int FI);
+ bool reserveVGPRforSGPRSpills(MachineFunction &MF);
bool allocateVGPRSpillToAGPR(MachineFunction &MF, int FI, bool isAGPRtoVGPR);
void removeDeadFrameIndices(MachineFrameInfo &MFI);
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 07d5b62cd250..08d6c97d8158 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -320,6 +320,10 @@ BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
for (MCPhysReg Reg : MFI->getVGPRSpillAGPRs())
reserveRegisterTuples(Reserved, Reg);
+ if (MFI->VGPRReservedForSGPRSpill)
+ for (auto SSpill : MFI->getSGPRSpillVGPRs())
+ reserveRegisterTuples(Reserved, SSpill.VGPR);
+
return Reserved;
}
@@ -1529,17 +1533,23 @@ bool SIRegisterInfo::shouldRewriteCopySrc(
return getCommonSubClass(DefRC, SrcRC) != nullptr;
}
-/// Returns a register that is not used at any point in the function.
+/// Returns a lowest register that is not used at any point in the function.
/// If all registers are used, then this function will return
-// AMDGPU::NoRegister.
-MCRegister
-SIRegisterInfo::findUnusedRegister(const MachineRegisterInfo &MRI,
- const TargetRegisterClass *RC,
- const MachineFunction &MF) const {
-
- for (MCRegister Reg : *RC)
- if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg))
- return Reg;
+/// AMDGPU::NoRegister. If \p ReserveHighestVGPR = true, then return
+/// highest unused register.
+MCRegister SIRegisterInfo::findUnusedRegister(const MachineRegisterInfo &MRI,
+ const TargetRegisterClass *RC,
+ const MachineFunction &MF,
+ bool ReserveHighestVGPR) const {
+ if (ReserveHighestVGPR) {
+ for (MCRegister Reg : reverse(*RC))
+ if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg))
+ return Reg;
+ } else {
+ for (MCRegister Reg : *RC)
+ if (MRI.isAllocatable(Reg) && !MRI.isPhysRegUsed(Reg))
+ return Reg;
+ }
return MCRegister();
}
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
index 2f328e77506e..bafd1472f373 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -196,7 +196,8 @@ class SIRegisterInfo final : public AMDGPUGenRegisterInfo {
MCRegister findUnusedRegister(const MachineRegisterInfo &MRI,
const TargetRegisterClass *RC,
- const MachineFunction &MF) const;
+ const MachineFunction &MF,
+ bool ReserveHighestVGPR = false) const;
const TargetRegisterClass *getRegClassForReg(const MachineRegisterInfo &MRI,
Register Reg) const;
diff --git a/llvm/test/CodeGen/AMDGPU/reserve-vgpr-for-sgpr-spill.ll b/llvm/test/CodeGen/AMDGPU/reserve-vgpr-for-sgpr-spill.ll
new file mode 100644
index 000000000000..3858fd9eec06
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/reserve-vgpr-for-sgpr-spill.ll
@@ -0,0 +1,50 @@
+; RUN: llc -mtriple amdgcn-amd-amdhsa -mcpu=gfx803 -O0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+
+define void @child_function() #0 {
+ call void asm sideeffect "", "~{vcc}" () #0
+ ret void
+}
+
+; GCN-LABEL: {{^}}parent_func:
+; CHECK: v_writelane_b32 v255, s33, 2
+; CHECK: v_writelane_b32 v255, s30, 0
+; CHECK: v_writelane_b32 v255, s31, 1
+; CHECK: s_swappc_b64 s[30:31], s[4:5]
+; CHECK: v_readlane_b32 s4, v255, 0
+; CHECK: v_readlane_b32 s5, v255, 1
+; CHECK: v_readlane_b32 s33, v255, 2
+; GCN: ; NumVgprs: 256
+
+define void @parent_func() #0 {
+ call void asm sideeffect "",
+ "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9}
+ ,~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19}
+ ,~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29}
+ ,~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}
+ ,~{v40},~{v41},~{v42},~{v43},~{v44},~{v45},~{v46},~{v47},~{v48},~{v49}
+ ,~{v50},~{v51},~{v52},~{v53},~{v54},~{v55},~{v56},~{v57},~{v58},~{v59}
+ ,~{v60},~{v61},~{v62},~{v63},~{v64},~{v65},~{v66},~{v67},~{v68},~{v69}
+ ,~{v70},~{v71},~{v72},~{v73},~{v74},~{v75},~{v76},~{v77},~{v78},~{v79}
+ ,~{v80},~{v81},~{v82},~{v83},~{v84},~{v85},~{v86},~{v87},~{v88},~{v89}
+ ,~{v90},~{v91},~{v92},~{v93},~{v94},~{v95},~{v96},~{v97},~{v98},~{v99}
+ ,~{v100},~{v101},~{v102},~{v103},~{v104},~{v105},~{v106},~{v107},~{v108},~{v109}
+ ,~{v110},~{v111},~{v112},~{v113},~{v114},~{v115},~{v116},~{v117},~{v118},~{v119}
+ ,~{v120},~{v121},~{v122},~{v123},~{v124},~{v125},~{v126},~{v127},~{v128},~{v129}
+ ,~{v130},~{v131},~{v132},~{v133},~{v134},~{v135},~{v136},~{v137},~{v138},~{v139}
+ ,~{v140},~{v141},~{v142},~{v143},~{v144},~{v145},~{v146},~{v147},~{v148},~{v149}
+ ,~{v150},~{v151},~{v152},~{v153},~{v154},~{v155},~{v156},~{v157},~{v158},~{v159}
+ ,~{v160},~{v161},~{v162},~{v163},~{v164},~{v165},~{v166},~{v167},~{v168},~{v169}
+ ,~{v170},~{v171},~{v172},~{v173},~{v174},~{v175},~{v176},~{v177},~{v178},~{v179}
+ ,~{v180},~{v181},~{v182},~{v183},~{v184},~{v185},~{v186},~{v187},~{v188},~{v189}
+ ,~{v190},~{v191},~{v192},~{v193},~{v194},~{v195},~{v196},~{v197},~{v198},~{v199}
+ ,~{v200},~{v201},~{v202},~{v203},~{v204},~{v205},~{v206},~{v207},~{v208},~{v209}
+ ,~{v210},~{v211},~{v212},~{v213},~{v214},~{v215},~{v216},~{v217},~{v218},~{v219}
+ ,~{v220},~{v221},~{v222},~{v223},~{v224},~{v225},~{v226},~{v227},~{v228},~{v229}
+ ,~{v230},~{v231},~{v232},~{v233},~{v234},~{v235},~{v236},~{v237},~{v238},~{v239}
+ ,~{v240},~{v241},~{v242},~{v243},~{v244},~{v245},~{v246},~{v247},~{v248},~{v249}
+ ,~{v250},~{v251},~{v252},~{v253},~{v254}" () #0
+ call void @child_function()
+ ret void
+}
+
+attributes #0 = { nounwind noinline norecurse }
More information about the llvm-commits
mailing list