[llvm-branch-commits] [llvm] [AMDGPU] Limit register pressure of pipelined loops (PR #212539)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Jul 29 07:53:04 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: hjagasiaAMD
<details>
<summary>Changes</summary>
Opt AMDGPU into the generic MachinePipeliner register-pressure detector via
shouldLimitRegPressure(), and supply an occupancy-aware verdict in
isScheduleRegPressureTooHigh(): reject a schedule whose SGPR or VGPR/AGPR
pressure would drop the kernel below its target occupancy, or exceed a
register class's addressability cap. On gfx90a+ VGPRs and AGPRs share one
register file, so their combined footprint is bounded together. These match
the limits GCNSchedStrategy enforces.
---
Full diff: https://github.com/llvm/llvm-project/pull/212539.diff
2 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/SIInstrInfo.cpp (+46-1)
- (added) llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-regpressure-retry.mir (+175)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index fd74399ca97c5..91f0524178a44 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -16,6 +16,7 @@
#include "AMDGPUInstrInfo.h"
#include "AMDGPULaneMaskUtils.h"
#include "GCNHazardRecognizer.h"
+#include "GCNRegPressure.h"
#include "GCNSubtarget.h"
#include "SIMachineFunctionInfo.h"
#include "Utils/AMDGPUBaseInfo.h"
@@ -3297,11 +3298,13 @@ class AMDGPUPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
const MachineInstr *CmpInst;
/// The normalized condition used by createTripCountGreaterCondition()
SmallVector<MachineOperand, 3> Cond;
+ MachineFunction *MF;
public:
AMDGPUPipelinerLoopInfo(MachineInstr *CmpInst,
const SmallVectorImpl<MachineOperand> &Cond)
- : CmpInst(CmpInst), Cond(Cond.begin(), Cond.end()) {}
+ : CmpInst(CmpInst), Cond(Cond.begin(), Cond.end()),
+ MF(CmpInst->getParent()->getParent()) {}
bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
return CmpInst && MI == CmpInst;
@@ -3311,6 +3314,48 @@ class AMDGPUPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
// cap that still rejects pathologically large ones.
std::optional<unsigned> getMaxMII() const override { return 256; }
+ bool shouldLimitRegPressure() const override { return true; }
+
+ // Reject a schedule needing more registers than the target occupancy allows.
+ std::optional<bool> isScheduleRegPressureTooHigh(
+ ArrayRef<unsigned> MaxSetPressure) const override {
+ const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
+ const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
+ unsigned TargetOcc = MFI->getOccupancy();
+
+ unsigned SGPRPressure =
+ MaxSetPressure[AMDGPU::RegisterPressureSets::SReg_32];
+ unsigned MaxSGPRs =
+ std::min(ST.getMaxNumSGPRs(TargetOcc, /*Addressable=*/true),
+ ST.getMaxNumSGPRs(*MF));
+ if (SGPRPressure > MaxSGPRs)
+ return true;
+
+ unsigned VGPRPressure =
+ MaxSetPressure[AMDGPU::RegisterPressureSets::VGPR_32];
+ unsigned AGPRPressure =
+ MaxSetPressure[AMDGPU::RegisterPressureSets::AGPR_32];
+
+ // The maximum number of arch VGPRs on a non-unified register file, or the
+ // maximum VGPR + AGPR in the unified (gfx90a+) register file case.
+ unsigned MaxVGPRs =
+ std::min(ST.getMaxNumVGPRs(TargetOcc, MFI->getDynamicVGPRBlockSize()),
+ ST.getMaxNumVGPRs(*MF));
+ unsigned CombinedVGPRs =
+ ST.hasGFX90AInsts()
+ ? GCNRegPressure::getUnifiedVGPRNum(VGPRPressure, AGPRPressure,
+ /*NumAVGPRs=*/0)
+ : std::max(VGPRPressure, AGPRPressure);
+ if (CombinedVGPRs > MaxVGPRs)
+ return true;
+
+ // The maximum number of arch VGPRs for both unified and non-unified
+ // register files. Each class must fit this cap, which is tighter than the
+ // combined budget at low occupancy.
+ unsigned MaxArchVGPRs = std::min(MaxVGPRs, ST.getAddressableNumArchVGPRs());
+ return VGPRPressure > MaxArchVGPRs || AGPRPressure > MaxArchVGPRs;
+ }
+
std::optional<bool> createTripCountGreaterCondition(
int TC, MachineBasicBlock &MBB,
SmallVectorImpl<MachineOperand> &CondParam) override {
diff --git a/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-regpressure-retry.mir b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-regpressure-retry.mir
new file mode 100644
index 0000000000000..5eccb86d1832e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-regpressure-retry.mir
@@ -0,0 +1,175 @@
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=pipeliner -debug-only=pipeliner -o /dev/null %s 2>&1 | FileCheck %s
+# REQUIRES: asserts
+
+# Reduced from the Triton _batched_gemm_a8w8_kernel, where pipelining a
+# too-high-pressure schedule regressed gfx942 performance by ~80%.
+#
+# Check that a schedule needing more VGPRs than the target occupancy allows is
+# rejected on register pressure and the II search retries. occupancy: 8 sets a
+# combined VGPR+AGPR budget of 512/8 = 64
+
+# CHECK: Rejected the schedule because of too high register pressure (per target verdict)
+# CHECK: Schedule Found? 1
+
+---
+name: swp_amdgpu_pipeline_regpressure_retry
+tracksRegLiveness: true
+isSSA: true
+machineFunctionInfo:
+ isEntryFunction: true
+ occupancy: 8
+body: |
+ bb.0:
+ %0:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ %1:sreg_32 = S_MOV_B32 0
+ %2:vreg_64_align2 = AV_MOV_B64_IMM_PSEUDO 0, implicit $exec
+ %3:vreg_64_align2 = AV_MOV_B64_IMM_PSEUDO 1, implicit $exec
+
+ bb.1:
+ successors: %bb.2, %bb.1
+
+ %4:sreg_32 = PHI %1, %bb.0, %5, %bb.1
+ %6:av_32 = PHI %0, %bb.0, %7, %bb.1
+ %8:av_32 = PHI %0, %bb.0, %9, %bb.1
+ %10:av_32 = PHI %0, %bb.0, %11, %bb.1
+ %12:av_32 = PHI %0, %bb.0, %13, %bb.1
+ %14:av_32 = PHI %0, %bb.0, %15, %bb.1
+ %16:av_32 = PHI %0, %bb.0, %17, %bb.1
+ %18:av_32 = PHI %0, %bb.0, %19, %bb.1
+ %20:av_32 = PHI %0, %bb.0, %21, %bb.1
+ %22:av_32 = PHI %0, %bb.0, %23, %bb.1
+ %24:av_32 = PHI %0, %bb.0, %25, %bb.1
+ %26:av_32 = PHI %0, %bb.0, %27, %bb.1
+ %28:av_32 = PHI %0, %bb.0, %29, %bb.1
+ %30:av_32 = PHI %0, %bb.0, %31, %bb.1
+ %32:av_32 = PHI %0, %bb.0, %33, %bb.1
+ %34:av_32 = PHI %0, %bb.0, %35, %bb.1
+ %36:av_32 = PHI %0, %bb.0, %37, %bb.1
+ %38:av_32 = PHI %0, %bb.0, %39, %bb.1
+ %40:av_32 = PHI %0, %bb.0, %41, %bb.1
+ %42:av_32 = PHI %0, %bb.0, %43, %bb.1
+ %44:av_32 = PHI %0, %bb.0, %45, %bb.1
+ %46:vgpr_32 = PHI %0, %bb.0, %47, %bb.1
+ %48:vgpr_32 = PHI %0, %bb.0, %49, %bb.1
+ %50:vgpr_32 = PHI %0, %bb.0, %51, %bb.1
+ %52:vgpr_32 = PHI %0, %bb.0, %53, %bb.1
+ %54:av_32 = PHI %0, %bb.0, %55, %bb.1
+ %56:av_32 = PHI %0, %bb.0, %57, %bb.1
+ %58:av_32 = PHI %0, %bb.0, %59, %bb.1
+ %60:av_32 = PHI %0, %bb.0, %61, %bb.1
+ %62:av_32 = PHI %0, %bb.0, %63, %bb.1
+ %64:av_32 = PHI %0, %bb.0, %65, %bb.1
+ %66:av_32 = PHI %0, %bb.0, %67, %bb.1
+ %68:av_32 = PHI %0, %bb.0, %69, %bb.1
+ %70:av_32 = PHI %0, %bb.0, %71, %bb.1
+ %72:av_32 = PHI %0, %bb.0, %73, %bb.1
+ %74:vreg_64_align2 = REG_SEQUENCE %70, %subreg.sub0, %72, %subreg.sub1
+ %75:vreg_64_align2 = REG_SEQUENCE %66, %subreg.sub0, %68, %subreg.sub1
+ %76:vreg_64_align2 = REG_SEQUENCE %62, %subreg.sub0, %64, %subreg.sub1
+ %77:vreg_64_align2 = REG_SEQUENCE %58, %subreg.sub0, %60, %subreg.sub1
+ %78:vreg_64_align2 = REG_SEQUENCE %54, %subreg.sub0, %56, %subreg.sub1
+ %79:vreg_64_align2 = REG_SEQUENCE %50, %subreg.sub0, %52, %subreg.sub1
+ %80:vreg_64_align2 = REG_SEQUENCE %46, %subreg.sub0, %48, %subreg.sub1
+ %81:vreg_64_align2 = REG_SEQUENCE %42, %subreg.sub0, %44, %subreg.sub1
+ %82:vreg_64_align2 = REG_SEQUENCE %38, %subreg.sub0, %40, %subreg.sub1
+ %83:vreg_64_align2 = REG_SEQUENCE %34, %subreg.sub0, %36, %subreg.sub1
+ %84:vreg_64_align2 = REG_SEQUENCE %30, %subreg.sub0, %32, %subreg.sub1
+ %85:vreg_64_align2 = REG_SEQUENCE %26, %subreg.sub0, %28, %subreg.sub1
+ %86:vreg_64_align2 = REG_SEQUENCE %22, %subreg.sub0, %24, %subreg.sub1
+ %87:vreg_64_align2 = REG_SEQUENCE %18, %subreg.sub0, %20, %subreg.sub1
+ %88:vreg_64_align2 = REG_SEQUENCE %14, %subreg.sub0, %16, %subreg.sub1
+ %89:vreg_64_align2 = REG_SEQUENCE %10, %subreg.sub0, %12, %subreg.sub1
+ %90:vreg_64_align2 = REG_SEQUENCE %6, %subreg.sub0, %8, %subreg.sub1
+ S_WAITCNT .Vmcnt_0_Expcnt_0_Lgkmcnt_0
+ %91:vreg_128_align2 = DS_READ2ST64_B64_gfx9 %0, 0, 4, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) inttoptr (i32 2048 to ptr addrspace(3))`, addrspace 3), (load (s64) from `ptr addrspace(3) null`, addrspace 3)
+ %92:vreg_128_align2 = DS_READ2ST64_B64_gfx9 %0, 8, 24, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) inttoptr (i32 12288 to ptr addrspace(3))`, addrspace 3), (load (s64) from `ptr addrspace(3) inttoptr (i32 4096 to ptr addrspace(3))`, addrspace 3)
+ %93:vreg_128_align2 = V_MFMA_I32_16X16X32I8_vgprcd_e64 %2, %91.sub2_sub3, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %94:vreg_128_align2 = V_MFMA_I32_16X16X32I8_vgprcd_e64 %2, %2, killed %93, 0, 0, 0, implicit $mode, implicit $exec
+ %95:vgpr_32 = V_CVT_F32_I32_e32 %94.sub1, implicit $mode, implicit $exec
+ %96:vgpr_32 = V_CVT_F32_I32_e32 %94.sub0, implicit $mode, implicit $exec
+ %97:vreg_64_align2 = REG_SEQUENCE killed %96, %subreg.sub0, killed %95, %subreg.sub1
+ %98:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, killed %74, 8, %97, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %99:vgpr_32 = V_CVT_F32_I32_e32 %94.sub3, implicit $mode, implicit $exec
+ %100:vgpr_32 = V_CVT_F32_I32_e32 %94.sub2, implicit $mode, implicit $exec
+ %101:vreg_64_align2 = REG_SEQUENCE killed %100, %subreg.sub0, killed %99, %subreg.sub1
+ %102:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, killed %75, 8, %101, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %103:vreg_128_align2 = V_MFMA_I32_16X16X32I8_vgprcd_e64 %92.sub2_sub3, %2, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %104:vreg_128_align2 = V_MFMA_I32_16X16X32I8_vgprcd_e64 %2, %2, killed %103, 0, 0, 0, implicit $mode, implicit $exec
+ %105:vgpr_32 = V_CVT_F32_I32_e32 %104.sub1, implicit $mode, implicit $exec
+ %106:vgpr_32 = V_CVT_F32_I32_e32 %104.sub0, implicit $mode, implicit $exec
+ %107:vreg_64_align2 = REG_SEQUENCE killed %106, %subreg.sub0, killed %105, %subreg.sub1
+ %108:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, killed %76, 8, %107, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %109:vreg_128_align2 = V_MFMA_I32_16X16X32I8_vgprcd_e64 %91.sub0_sub1, %2, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %110:vreg_128_align2 = V_MFMA_I32_16X16X32I8_vgprcd_e64 %2, %2, killed %109, 0, 0, 0, implicit $mode, implicit $exec
+ %111:vgpr_32 = V_CVT_F32_I32_e32 %110.sub1, implicit $mode, implicit $exec
+ %112:vgpr_32 = V_CVT_F32_I32_e32 %110.sub0, implicit $mode, implicit $exec
+ %113:vreg_64_align2 = REG_SEQUENCE killed %112, %subreg.sub0, killed %111, %subreg.sub1
+ %114:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, killed %77, 8, %113, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %115:vgpr_32 = V_CVT_F32_I32_e32 %110.sub3, implicit $mode, implicit $exec
+ %116:vgpr_32 = V_CVT_F32_I32_e32 %110.sub2, implicit $mode, implicit $exec
+ %117:vreg_64_align2 = REG_SEQUENCE killed %116, %subreg.sub0, killed %115, %subreg.sub1
+ %118:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, killed %78, 8, %117, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %119:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, killed %79, 0, 1065353216, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %120:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, killed %80, 0, 1065353216, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %121:vreg_128_align2 = V_MFMA_I32_16X16X32I8_vgprcd_e64 %3, %92.sub0_sub1, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %122:vreg_128_align2 = V_MFMA_I32_16X16X32I8_vgprcd_e64 %2, %2, killed %121, 0, 0, 0, implicit $mode, implicit $exec
+ %123:vgpr_32 = V_CVT_F32_I32_e32 %122.sub3, implicit $mode, implicit $exec
+ %124:vgpr_32 = V_CVT_F32_I32_e32 %122.sub2, implicit $mode, implicit $exec
+ %125:vreg_64_align2 = REG_SEQUENCE killed %124, %subreg.sub0, killed %123, %subreg.sub1
+ %126:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, killed %81, 8, %125, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %127:vreg_128_align2 = V_MFMA_I32_16X16X32I8_vgprcd_e64 %2, %92.sub0_sub1, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %128:vreg_128_align2 = V_MFMA_I32_16X16X32I8_vgprcd_e64 %2, %2, killed %127, 0, 0, 0, implicit $mode, implicit $exec
+ %129:vgpr_32 = V_CVT_F32_I32_e32 %128.sub1, implicit $mode, implicit $exec
+ %130:vgpr_32 = V_CVT_F32_I32_e32 %128.sub0, implicit $mode, implicit $exec
+ %131:vreg_64_align2 = REG_SEQUENCE killed %130, %subreg.sub0, killed %129, %subreg.sub1
+ %132:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, killed %82, 8, %131, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %133:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, killed %83, 8, %97, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %134:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, killed %84, 8, %101, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %135:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, killed %85, 8, %107, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %136:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, killed %86, 8, %113, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %137:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, killed %87, 8, %117, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %138:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, killed %88, 8, %125, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %139:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, killed %89, 8, %131, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ %140:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, killed %90, 8, %97, 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ S_CMP_LG_U32 %4, 0, implicit-def $scc
+ %7:av_32 = COPY %140.sub0
+ %9:av_32 = COPY %140.sub1
+ %11:av_32 = COPY %139.sub0
+ %13:av_32 = COPY %139.sub1
+ %15:av_32 = COPY %138.sub0
+ %17:av_32 = COPY %138.sub1
+ %19:av_32 = COPY %137.sub0
+ %21:av_32 = COPY %137.sub1
+ %23:av_32 = COPY %136.sub0
+ %25:av_32 = COPY %136.sub1
+ %27:av_32 = COPY %135.sub0
+ %29:av_32 = COPY %135.sub1
+ %31:av_32 = COPY %134.sub0
+ %33:av_32 = COPY %134.sub1
+ %35:av_32 = COPY %133.sub0
+ %37:av_32 = COPY %133.sub1
+ %39:av_32 = COPY %132.sub0
+ %41:av_32 = COPY %132.sub1
+ %43:av_32 = COPY %126.sub0
+ %45:av_32 = COPY %126.sub1
+ %47:vgpr_32 = COPY %120.sub0
+ %49:vgpr_32 = COPY %120.sub1
+ %51:vgpr_32 = COPY %119.sub0
+ %53:vgpr_32 = COPY %119.sub1
+ %55:av_32 = COPY %118.sub0
+ %57:av_32 = COPY %118.sub1
+ %59:av_32 = COPY %114.sub0
+ %61:av_32 = COPY %114.sub1
+ %63:av_32 = COPY %108.sub0
+ %65:av_32 = COPY %108.sub1
+ %67:av_32 = COPY %102.sub0
+ %69:av_32 = COPY %102.sub1
+ %71:av_32 = COPY %98.sub0
+ %73:av_32 = COPY %98.sub1
+ %5:sreg_32 = S_MOV_B32 1
+ S_CBRANCH_SCC1 %bb.1, implicit $scc
+ S_BRANCH %bb.2
+
+ bb.2:
+ S_ENDPGM 0
+...
``````````
</details>
https://github.com/llvm/llvm-project/pull/212539
More information about the llvm-branch-commits
mailing list