[llvm-branch-commits] [llvm] [AMDGPU] Apply occupancy-aware register allocation anti-hints (PR #218074)

[Lukas Sommer via llvm-branch-commits]

================
@@ -4173,6 +4178,130 @@ bool SIRegisterInfo::getRegAllocationHints(Register VirtReg,
   }
 }
 
+bool SIRegisterInfo::shouldApplyAntiHints(
+    const MachineFunction &MF, unsigned NumAllocatedVGPRs,
+    unsigned &MaxVGPRsForCurrentOccupancy) const {
+
+  const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+  unsigned DynamicVGPRBlockSize = MFI->getDynamicVGPRBlockSize();
+  unsigned TargetOccupancy = MFI->getOccupancy();
+  unsigned CurrentOccupancy =
+      ST.getOccupancyWithNumVGPRs(NumAllocatedVGPRs, DynamicVGPRBlockSize);
+  MaxVGPRsForCurrentOccupancy =
+      ST.getMaxNumVGPRs(CurrentOccupancy, DynamicVGPRBlockSize);
+
+  LLVM_DEBUG(dbgs() << "anti-hints: " << NumAllocatedVGPRs
+                    << " VGPRs allocated, target occupancy " << TargetOccupancy
+                    << ", current occupancy " << CurrentOccupancy << '\n');
+
+  // If we are already at lowest occupancy, then there is no need to protect
+  // against occupancy regression.
+  if (CurrentOccupancy == 1)
+    return true;
+
+  // Set max VGPRs for target and current occupancy to early bail out if we are
+  // close to the limit.
+  unsigned MaxVGPRsCutOffForTargetOccupancy =
+      (ST.getMaxNumVGPRs(TargetOccupancy, DynamicVGPRBlockSize) * 80) / 100;
+  unsigned MaxVGPRsCutOffForCurrentOccupancy =
+      (MaxVGPRsForCurrentOccupancy * 95) / 100;
+
+  if (NumAllocatedVGPRs >= MaxVGPRsCutOffForTargetOccupancy) {
+    LLVM_DEBUG(dbgs() << "anti-hints: not applied, at or above the "
+                      << MaxVGPRsCutOffForTargetOccupancy
+                      << " VGPR cutoff for target occupancy\n");
+    return false;
+  }
+
+  if (NumAllocatedVGPRs >= MaxVGPRsCutOffForCurrentOccupancy) {
+    LLVM_DEBUG(dbgs() << "anti-hints: not applied, at or above the "
+                      << MaxVGPRsCutOffForCurrentOccupancy
+                      << " VGPR cutoff for current occupancy\n");
+    return false;
+  }
+
+  return true;
+}
+
+void SIRegisterInfo::applyRegAllocationAntiHints(
+    Register VirtReg, ArrayRef<MCPhysReg> &Order,
+    SmallVectorImpl<MCPhysReg> &OrderStorage,
+    SmallVectorImpl<MCPhysReg> &AntiHints, const MachineFunction &MF,
+    const VirtRegMap *VRM, const LiveRegMatrix *Matrix) const {
+
+  // Early exit to default order if we have no anti-hints or no VRM.
+  if (AntiHints.empty() || !VRM)
+    return;
+
+  // Get total number of allocated VGPRs to determine the current occupancy.
+  unsigned NumVGPRs = 0;
+  unsigned NumAGPRs = 0;
+  if (Matrix) {
+    for (MCPhysReg Reg : AMDGPU::VGPR_32RegClass)
+      if (Matrix->isPhysRegUsed(Reg))
+        NumVGPRs = std::max(NumVGPRs, getHWRegIndex(Reg) + 1);
+    for (MCPhysReg Reg : AMDGPU::AGPR_32RegClass)
+      if (Matrix->isPhysRegUsed(Reg))
+        NumAGPRs = std::max(NumAGPRs, getHWRegIndex(Reg) + 1);
+  }
+  unsigned NumAllocatedVGPRs =
+      AMDGPU::getTotalNumVGPRs(ST.hasGFX90AInsts(), NumAGPRs, NumVGPRs);
+
+  // Early exit if we should not apply anti-hints.
+  unsigned MaxVGPRsForCurrentOccupancy = 0;
+  if (!shouldApplyAntiHints(MF, NumAllocatedVGPRs, MaxVGPRsForCurrentOccupancy))
+    return;
+
+  // Returns true if Reg fits within the current occupancy VGPR budget.
+  auto IsWithinBudget = [&](MCPhysReg Reg) -> bool {
+    const TargetRegisterClass *RC = getPhysRegBaseClass(Reg);
+
+    // No VGPR or AGPR usage.
+    if (!RC ||
+        (!isVGPRClass(RC) && !isAGPRClass(RC) && !isVectorSuperClass(RC)))
+      return true;
+
+    unsigned NumRegs = divideCeil(getRegSizeInBits(*RC), 32);
+    unsigned Highest = getHWRegIndex(Reg) + NumRegs - 1;
+    return Highest < MaxVGPRsForCurrentOccupancy;
----------------
sommerlukas wrote:

Is this correct on GFX90A? For an AGPR, wouldn't the index of the AGPR need to be added to the highest VGPR in that case?

https://github.com/llvm/llvm-project/pull/218074