[llvm] [AMDGPU] Limit DVGPR entry function max VGPRs to block size (PR #226242)
Mirko BrkuĊĦanin via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 25 09:56:36 PDT 2026
https://github.com/mbrkusanin updated https://github.com/llvm/llvm-project/pull/226242
>From 505fcdc9da3acfd8157baa82105207279b8c187d Mon Sep 17 00:00:00 2001
From: Mirko Brkusanin <Mirko.Brkusanin at amd.com>
Date: Thu, 24 Sep 2026 19:28:17 +0200
Subject: [PATCH 1/2] [AMDGPU] Limit DVGPR entry function max VGPRs to block
size
---
llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp | 16 ++
.../lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 5 +
llvm/lib/Target/AMDGPU/GCNSubtarget.cpp | 11 +-
...gpr-vgpr-count-propagation-direct-chain.ll | 7 +-
.../AMDGPU/dvgpr-vgpr-count-propagation.ll | 7 +-
.../dynamic-vgpr-entry-register-pressure.ll | 199 ++++++++++++++++++
.../AMDGPU/dynamic-vgpr-entry-vgpr-limit.ll | 28 +++
.../dynamic-vgpr-reserve-stack-for-cwsr.ll | 28 +--
.../AMDGPU/promote-alloca-vgpr-ratio.ll | 41 +++-
9 files changed, 317 insertions(+), 25 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/dynamic-vgpr-entry-register-pressure.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/dynamic-vgpr-entry-vgpr-limit.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index c18b840c36791..26787724ac23c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -1409,6 +1409,22 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
MF.getFunction(), "local memory", MFI->getLDSSize(),
STM.getAddressableLocalMemorySize(), DS_Error));
}
+
+ // Catches the paths the register allocator budget cannot constrain: explicit
+ // physical registers in inline asm and wave dispatch VGPR arguments.
+ if (MFI->isDynamicVGPREnabled() &&
+ AMDGPU::isEntryFunctionCC(F.getCallingConv())) {
+ unsigned BlockSize = MFI->getDynamicVGPRBlockSize();
+ uint64_t NumVgpr;
+ if (TryGetMCExprValue(ProgInfo.NumVGPRsForWavesPerEU, NumVgpr) &&
+ NumVgpr > BlockSize) {
+ LLVMContext &Ctx = F.getContext();
+ Ctx.diagnose(DiagnosticInfoResourceLimit(
+ F, "dynamic VGPR entry point vector registers", NumVgpr, BlockSize,
+ DS_Error, DK_ResourceLimit));
+ }
+ }
+
// The MCExpr equivalent of getNumSGPRBlocks/getNumVGPRBlocks:
// (alignTo(max(1u, NumGPR), GPREncodingGranule) / GPREncodingGranule) - 1
auto GetNumGPRBlocks = [&CreateExpr, &Ctx](const MCExpr *NumGPR,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index aefcabd0d3bf1..4e934ac2f0aad 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -218,6 +218,11 @@ static unsigned getMaxVGPRs(unsigned LDSBytes, const TargetMachine &TM,
ST.getWavesPerEU(ST.getFlatWorkGroupSizes(F), LDSBytes, F).first,
DynamicVGPRBlockSize);
+ // A DVGPR wave launches with a single VGPR block allocated.
+ if (DynamicVGPRBlockSize != 0 &&
+ AMDGPU::isEntryFunctionCC(F.getCallingConv()))
+ MaxVGPRs = std::min(MaxVGPRs, DynamicVGPRBlockSize);
+
// A non-entry function has only 32 caller preserved registers.
// Do not promote alloca which will force spilling unless we know the function
// will be inlined.
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp b/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
index 34d987cd6d266..a96e4d048adce 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
@@ -636,9 +636,18 @@ unsigned GCNSubtarget::getBaseMaxNumVGPRs(
unsigned GCNSubtarget::getMaxNumVGPRs(const Function &F) const {
unsigned DynamicVGPRBlockSize = AMDGPU::getDynamicVGPRBlockSize(F);
std::pair<unsigned, unsigned> Waves = getWavesPerEU(F);
- return getBaseMaxNumVGPRs(
+
+ unsigned MaxNumVGPRs = getBaseMaxNumVGPRs(
F, {getMinNumVGPRs(Waves.second, DynamicVGPRBlockSize),
getMaxNumVGPRs(Waves.first, DynamicVGPRBlockSize)});
+
+ // In DVGPR mode, a wave launches with a single VGPR block allocated. Applied
+ // after getBaseMaxNumVGPRs so "amdgpu-num-vgpr" cannot raise it back up.
+ if (DynamicVGPRBlockSize != 0 &&
+ AMDGPU::isEntryFunctionCC(F.getCallingConv()))
+ MaxNumVGPRs = std::min(MaxNumVGPRs, DynamicVGPRBlockSize);
+
+ return MaxNumVGPRs;
}
unsigned GCNSubtarget::getMaxNumVGPRs(const MachineFunction &MF) const {
diff --git a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-direct-chain.ll b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-direct-chain.ll
index 88683131078cf..06551a03536eb 100644
--- a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-direct-chain.ll
+++ b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation-direct-chain.ll
@@ -11,7 +11,7 @@
; func.2 and func.4 have DVGPRs disabled
; DVGPR: .set .Lgfx_func_a.num_vgpr, 40
-; DVGPR: .set .Lamdgpu_cs_main.num_vgpr, max(42, .Lgfx_func_a.num_vgpr)
+; DVGPR: .set .Lamdgpu_cs_main.num_vgpr, 11
; DVGPR: .set .Lfunc.0.num_vgpr, max(11, .Lgfx_func_a.num_vgpr)
; DVGPR: .set .Lfunc.1.num_vgpr, 82
; DVGPR: .set .Lfunc.2.num_vgpr, max(11, amdgpu.max_num_vgpr)
@@ -22,7 +22,7 @@
; DVGPR: .set amdgpu.max_num_vgpr, 82
; NODVGPR: .set .Lgfx_func_a.num_vgpr, 40
-; NODVGPR: .set .Lamdgpu_cs_main.num_vgpr, max(42, amdgpu.max_num_vgpr)
+; NODVGPR: .set .Lamdgpu_cs_main.num_vgpr, max(11, amdgpu.max_num_vgpr)
; NODVGPR: .set .Lfunc.0.num_vgpr, max(11, amdgpu.max_num_vgpr)
; NODVGPR: .set .Lfunc.1.num_vgpr, max(82, amdgpu.max_num_vgpr)
; NODVGPR: .set .Lfunc.2.num_vgpr, max(11, amdgpu.max_num_vgpr)
@@ -33,7 +33,7 @@
; NODVGPR: .set amdgpu.max_num_vgpr, 82
; DVGPR: - .hardware_stages:
-; DVGPR: .vgpr_count: 0x2a
+; DVGPR: .vgpr_count: 0xb
; DVGPR: .shader_functions:
; DVGPR: func.0:
; DVGPR: .vgpr_count: 0x28
@@ -71,7 +71,6 @@ define amdgpu_gfx void @gfx_func_a() #0 {
define amdgpu_cs void @amdgpu_cs_main(<3 x i32> inreg %sgprs, <3 x i32> %vgprs) #0 {
%fptr = load ptr, ptr inttoptr(i64 0 to ptr)
- call amdgpu_gfx void @gfx_func_a()
call void(ptr, i32, <3 x i32>, <3 x i32>, i32, ...) @llvm.amdgcn.cs.chain.v3i32(ptr inreg %fptr, i32 inreg 0, <3 x i32> inreg %sgprs, <3 x i32> zeroinitializer, i32 1, i32 0, i32 -1, ptr @func.1)
unreachable
}
diff --git a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation.ll b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation.ll
index 22ea185019cf9..81e6c1b7e1255 100644
--- a/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation.ll
+++ b/llvm/test/CodeGen/AMDGPU/dvgpr-vgpr-count-propagation.ll
@@ -14,7 +14,7 @@
; DVGPR: .set .Lgfx_func_a.num_vgpr, 40
; DVGPR: .set .Lgfx_func_b2.num_vgpr, 80
; DVGPR: .set .Lgfx_func_b.num_vgpr, max(61, .Lgfx_func_b2.num_vgpr)
-; DVGPR: .set .Lamdgpu_cs_main.num_vgpr, max(42, .Lgfx_func_a.num_vgpr)
+; DVGPR: .set .Lamdgpu_cs_main.num_vgpr, 11
; DVGPR: .set .Lfunc.0.num_vgpr, 13
; DVGPR: .set .Lfunc.1.num_vgpr, max(14, .Lgfx_func_a.num_vgpr, .Lgfx_func_b.num_vgpr)
; DVGPR: .set .Lfunc.2.num_vgpr, max(16, .Lgfx_func_a.num_vgpr)
@@ -27,7 +27,7 @@
; NODVGPR: .set .Lgfx_func_a.num_vgpr, 40
; NODVGPR: .set .Lgfx_func_b2.num_vgpr, 80
; NODVGPR: .set .Lgfx_func_b.num_vgpr, max(61, .Lgfx_func_b2.num_vgpr)
-; NODVGPR: .set .Lamdgpu_cs_main.num_vgpr, max(42, amdgpu.max_num_vgpr)
+; NODVGPR: .set .Lamdgpu_cs_main.num_vgpr, max(11, amdgpu.max_num_vgpr)
; NODVGPR: .set .Lfunc.0.num_vgpr, max(13, amdgpu.max_num_vgpr)
; NODVGPR: .set .Lfunc.1.num_vgpr, max(14, amdgpu.max_num_vgpr)
; NODVGPR: .set .Lfunc.2.num_vgpr, max(16, amdgpu.max_num_vgpr)
@@ -38,7 +38,7 @@
; NODVGPR: .set amdgpu.max_num_vgpr, 100
; DVGPR: - .hardware_stages:
-; DVGPR: .vgpr_count: 0x2a
+; DVGPR: .vgpr_count: 0xb
; DVGPR: .shader_functions:
; DVGPR: func.0:
; DVGPR: .vgpr_count: 0xd
@@ -95,7 +95,6 @@ define amdgpu_gfx void @gfx_func_b() #0 {
define amdgpu_cs void @amdgpu_cs_main(<3 x i32> inreg %sgprs, <3 x i32> %vgprs) #0 {
%fptr = load ptr, ptr inttoptr(i64 0 to ptr)
- call amdgpu_gfx void @gfx_func_a()
call void(ptr, i32, <3 x i32>, <3 x i32>, i32, ...) @llvm.amdgcn.cs.chain.v3i32(ptr inreg %fptr, i32 inreg 0, <3 x i32> inreg %sgprs, <3 x i32> zeroinitializer, i32 1, i32 0, i32 -1, ptr @func.1)
unreachable
}
diff --git a/llvm/test/CodeGen/AMDGPU/dynamic-vgpr-entry-register-pressure.ll b/llvm/test/CodeGen/AMDGPU/dynamic-vgpr-entry-register-pressure.ll
new file mode 100644
index 0000000000000..c3176031dab82
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/dynamic-vgpr-entry-register-pressure.ll
@@ -0,0 +1,199 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu12.00-amd-amdpal < %s | FileCheck %s
+
+; DVGPR entry point with register pressure that exceeds the single-block budget
+; has to spill some VGPRs.
+define amdgpu_cs void @dvgpr_spill_under_cap() #0 {
+; CHECK-LABEL: dvgpr_spill_under_cap:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_getreg_b32 s33, hwreg(HW_REG_WAVE_HW_ID2, 8, 2)
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v9
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: s_cmp_lg_u32 0, s33
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v10
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: s_cmovk_i32 s33, 0x1c0
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v11
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v12
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v13
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v14
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v15
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v0
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v1
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v2
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v3
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v8
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v7
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v6
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v5
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v4
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: scratch_store_b32 off, v4, s33 offset:16 ; 4-byte Folded Spill
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v4
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: scratch_store_b32 off, v4, s33 offset:12 ; 4-byte Folded Spill
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v4
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: scratch_store_b32 off, v4, s33 offset:8 ; 4-byte Folded Spill
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v4
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: scratch_store_b32 off, v4, s33 offset:4 ; 4-byte Folded Spill
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; def v4
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: scratch_store_b32 off, v4, s33 ; 4-byte Folded Spill
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; barrier
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use v9
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use v10
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use v11
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use v12
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use v13
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use v14
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use v15
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use v0
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use v1
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use v2
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use v3
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use v8
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use v7
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use v6
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use v5
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: scratch_load_b32 v0, off, s33 offset:16 th:TH_LOAD_LU ; 4-byte Folded Reload
+; CHECK-NEXT: s_wait_loadcnt 0x0
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use v0
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: scratch_load_b32 v0, off, s33 offset:12 th:TH_LOAD_LU ; 4-byte Folded Reload
+; CHECK-NEXT: s_wait_loadcnt 0x0
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use v0
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: scratch_load_b32 v0, off, s33 offset:8 th:TH_LOAD_LU ; 4-byte Folded Reload
+; CHECK-NEXT: s_wait_loadcnt 0x0
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use v0
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: scratch_load_b32 v0, off, s33 offset:4 th:TH_LOAD_LU ; 4-byte Folded Reload
+; CHECK-NEXT: s_wait_loadcnt 0x0
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use v0
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: scratch_load_b32 v0, off, s33 th:TH_LOAD_LU ; 4-byte Folded Reload
+; CHECK-NEXT: s_wait_loadcnt 0x0
+; CHECK-NEXT: ;;#ASMSTART
+; CHECK-NEXT: ; use v0
+; CHECK-NEXT: ;;#ASMEND
+; CHECK-NEXT: s_alloc_vgpr 0
+; CHECK-NEXT: s_endpgm
+; CHECK: NumVgprs: 16
+ %v0 = call i32 asm sideeffect "; def $0", "=v"()
+ %v1 = call i32 asm sideeffect "; def $0", "=v"()
+ %v2 = call i32 asm sideeffect "; def $0", "=v"()
+ %v3 = call i32 asm sideeffect "; def $0", "=v"()
+ %v4 = call i32 asm sideeffect "; def $0", "=v"()
+ %v5 = call i32 asm sideeffect "; def $0", "=v"()
+ %v6 = call i32 asm sideeffect "; def $0", "=v"()
+ %v7 = call i32 asm sideeffect "; def $0", "=v"()
+ %v8 = call i32 asm sideeffect "; def $0", "=v"()
+ %v9 = call i32 asm sideeffect "; def $0", "=v"()
+ %v10 = call i32 asm sideeffect "; def $0", "=v"()
+ %v11 = call i32 asm sideeffect "; def $0", "=v"()
+ %v12 = call i32 asm sideeffect "; def $0", "=v"()
+ %v13 = call i32 asm sideeffect "; def $0", "=v"()
+ %v14 = call i32 asm sideeffect "; def $0", "=v"()
+ %v15 = call i32 asm sideeffect "; def $0", "=v"()
+ %v16 = call i32 asm sideeffect "; def $0", "=v"()
+ %v17 = call i32 asm sideeffect "; def $0", "=v"()
+ %v18 = call i32 asm sideeffect "; def $0", "=v"()
+ %v19 = call i32 asm sideeffect "; def $0", "=v"()
+
+ call void asm sideeffect "; barrier", ""()
+
+ call void asm sideeffect "; use $0", "v"(i32 %v0)
+ call void asm sideeffect "; use $0", "v"(i32 %v1)
+ call void asm sideeffect "; use $0", "v"(i32 %v2)
+ call void asm sideeffect "; use $0", "v"(i32 %v3)
+ call void asm sideeffect "; use $0", "v"(i32 %v4)
+ call void asm sideeffect "; use $0", "v"(i32 %v5)
+ call void asm sideeffect "; use $0", "v"(i32 %v6)
+ call void asm sideeffect "; use $0", "v"(i32 %v7)
+ call void asm sideeffect "; use $0", "v"(i32 %v8)
+ call void asm sideeffect "; use $0", "v"(i32 %v9)
+ call void asm sideeffect "; use $0", "v"(i32 %v10)
+ call void asm sideeffect "; use $0", "v"(i32 %v11)
+ call void asm sideeffect "; use $0", "v"(i32 %v12)
+ call void asm sideeffect "; use $0", "v"(i32 %v13)
+ call void asm sideeffect "; use $0", "v"(i32 %v14)
+ call void asm sideeffect "; use $0", "v"(i32 %v15)
+ call void asm sideeffect "; use $0", "v"(i32 %v16)
+ call void asm sideeffect "; use $0", "v"(i32 %v17)
+ call void asm sideeffect "; use $0", "v"(i32 %v18)
+ call void asm sideeffect "; use $0", "v"(i32 %v19)
+ ret void
+}
+
+attributes #0 = { nounwind "amdgpu-dynamic-vgpr-block-size"="16" }
diff --git a/llvm/test/CodeGen/AMDGPU/dynamic-vgpr-entry-vgpr-limit.ll b/llvm/test/CodeGen/AMDGPU/dynamic-vgpr-entry-vgpr-limit.ll
new file mode 100644
index 0000000000000..8858d9c7ebf43
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/dynamic-vgpr-entry-vgpr-limit.ll
@@ -0,0 +1,28 @@
+; RUN: not llc -mtriple=amdgpu12.00-amd-amdpal -filetype=null < %s 2>&1 | FileCheck %s
+
+; CHECK: error: {{.*}}dynamic VGPR entry point vector registers (21) exceeds limit (16) in function 'entry_block16'
+define amdgpu_cs void @entry_block16() #0 {
+ call void asm sideeffect "", "~{v20}"()
+ ret void
+}
+
+; CHECK: error: {{.*}}dynamic VGPR entry point vector registers (21) exceeds limit (16) in function 'kernel_block16'
+define amdgpu_kernel void @kernel_block16() #0 {
+ call void asm sideeffect "", "~{v20}"()
+ ret void
+}
+
+; CHECK-NOT: error{{.*}}'entry_block32'
+define amdgpu_cs void @entry_block32() #1 {
+ call void asm sideeffect "", "~{v20}"()
+ ret void
+}
+
+; CHECK-NOT: error{{.*}}'chain_block16'
+define amdgpu_cs_chain void @chain_block16() #0 {
+ call void asm sideeffect "", "~{v20}"()
+ unreachable
+}
+
+attributes #0 = { nounwind "amdgpu-dynamic-vgpr-block-size"="16" }
+attributes #1 = { nounwind "amdgpu-dynamic-vgpr-block-size"="32" }
diff --git a/llvm/test/CodeGen/AMDGPU/dynamic-vgpr-reserve-stack-for-cwsr.ll b/llvm/test/CodeGen/AMDGPU/dynamic-vgpr-reserve-stack-for-cwsr.ll
index 06e71249c4732..4e7881b14730b 100644
--- a/llvm/test/CodeGen/AMDGPU/dynamic-vgpr-reserve-stack-for-cwsr.ll
+++ b/llvm/test/CodeGen/AMDGPU/dynamic-vgpr-reserve-stack-for-cwsr.ll
@@ -150,36 +150,36 @@ define amdgpu_cs void @with_spills() #0 {
ret void
}
-define amdgpu_cs void @realign_stack(<32 x i32> %x) #0 {
+define amdgpu_cs void @realign_stack(<16 x i32> %x) #0 {
; CHECK-LABEL: realign_stack:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_getreg_b32 s33, hwreg(HW_REG_WAVE_HW_ID2, 8, 2)
-; CHECK-NEXT: v_mov_b32_e32 v32, 0
-; CHECK-NEXT: s_cmp_lg_u32 0, s33
; CHECK-NEXT: s_mov_b32 s1, callee at abs32@hi
-; CHECK-NEXT: s_cmovk_i32 s33, 0x200
+; CHECK-NEXT: s_cmp_lg_u32 0, s33
; CHECK-NEXT: s_mov_b32 s0, callee at abs32@lo
-; CHECK-NEXT: scratch_store_b32 off, v32, s33 scope:SCOPE_SYS
+; CHECK-NEXT: s_cmovk_i32 s33, 0x200
+; CHECK-NEXT: s_movk_i32 s32, 0x100
+; CHECK-NEXT: scratch_store_b128 off, v[12:15], s33 offset:128 ; 16-byte Folded Spill
+; CHECK-NEXT: v_mov_b32_e32 v12, 0
+; CHECK-NEXT: s_cmovk_i32 s32, 0x300
+; CHECK-NEXT: s_wait_storecnt 0x0
+; CHECK-NEXT: scratch_store_b32 off, v12, s33 scope:SCOPE_SYS
; CHECK-NEXT: s_wait_storecnt 0x0
-; CHECK-NEXT: s_clause 0x7
-; CHECK-NEXT: scratch_store_b128 off, v[24:27], s33 offset:96
-; CHECK-NEXT: scratch_store_b128 off, v[28:31], s33 offset:112
-; CHECK-NEXT: scratch_store_b128 off, v[16:19], s33 offset:64
-; CHECK-NEXT: scratch_store_b128 off, v[20:23], s33 offset:80
; CHECK-NEXT: scratch_store_b128 off, v[8:11], s33 offset:32
-; CHECK-NEXT: scratch_store_b128 off, v[12:15], s33 offset:48
+; CHECK-NEXT: scratch_load_b128 v[8:11], off, s33 offset:128 th:TH_LOAD_LU ; 16-byte Folded Reload
+; CHECK-NEXT: s_wait_loadcnt 0x0
+; CHECK-NEXT: s_clause 0x2
+; CHECK-NEXT: scratch_store_b128 off, v[8:11], s33 offset:48
; CHECK-NEXT: scratch_store_b128 off, v[4:7], s33 offset:16
; CHECK-NEXT: scratch_store_b128 off, v[0:3], s33
; CHECK-NEXT: v_mov_b32_e32 v0, 0x47
-; CHECK-NEXT: s_movk_i32 s32, 0x100
-; CHECK-NEXT: s_cmovk_i32 s32, 0x300
; CHECK-NEXT: s_swappc_b64 s[30:31], s[0:1]
; CHECK-NEXT: s_alloc_vgpr 0
; CHECK-NEXT: s_endpgm
%v = alloca <32 x i32>, align 128, addrspace(5)
; use volatile store to avoid promotion of alloca to registers
store volatile i32 0, ptr addrspace(5) %v
- store <32 x i32> %x, ptr addrspace(5) %v
+ store <16 x i32> %x, ptr addrspace(5) %v
call amdgpu_gfx void @callee(i32 71)
ret void
}
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-vgpr-ratio.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-vgpr-ratio.ll
index 4f544162d7220..5e767db0d7b19 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-vgpr-ratio.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-vgpr-ratio.ll
@@ -271,6 +271,44 @@ define amdgpu_kernel void @i32_16_elements_attrib(ptr %out) #2 {
ret void
}
+define amdgpu_cs void @i32_16_elements_dvgpr(ptr %out) #3 {
+; BASE-LABEL: define amdgpu_cs void @i32_16_elements_dvgpr(
+; BASE-SAME: ptr [[OUT:%.*]]) #[[ATTR3:[0-9]+]] {
+; BASE-NEXT: [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; BASE-NEXT: [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; BASE-NEXT: [[C1:%.*]] = icmp uge i32 [[X]], 3
+; BASE-NEXT: [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; BASE-NEXT: [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; BASE-NEXT: [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; BASE-NEXT: [[ALLOCA:%.*]] = alloca [16 x i32], align 16, addrspace(5)
+; BASE-NEXT: call void @llvm.memset.p5.i32(ptr addrspace(5) [[ALLOCA]], i8 0, i32 64, i1 false)
+; BASE-NEXT: [[GEP_0:%.*]] = getelementptr inbounds [16 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 0
+; BASE-NEXT: [[GEP_1:%.*]] = getelementptr inbounds [16 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 15
+; BASE-NEXT: store i32 42, ptr addrspace(5) [[GEP_0]], align 4
+; BASE-NEXT: store i32 43, ptr addrspace(5) [[GEP_1]], align 4
+; BASE-NEXT: [[GEP:%.*]] = getelementptr inbounds [16 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[SEL2]]
+; BASE-NEXT: [[LOAD:%.*]] = load i32, ptr addrspace(5) [[GEP]], align 4
+; BASE-NEXT: store i32 [[LOAD]], ptr [[OUT]], align 4
+; BASE-NEXT: ret void
+;
+ %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+ %c1 = icmp uge i32 %x, 3
+ %c2 = icmp uge i32 %y, 3
+ %sel1 = select i1 %c1, i32 1, i32 2
+ %sel2 = select i1 %c2, i32 0, i32 %sel1
+ %alloca = alloca [16 x i32], align 16, addrspace(5)
+ call void @llvm.memset.p5.i32(ptr addrspace(5) %alloca, i8 0, i32 64, i1 false)
+ %gep.0 = getelementptr inbounds [16 x i32], ptr addrspace(5) %alloca, i32 0, i32 0
+ %gep.1 = getelementptr inbounds [16 x i32], ptr addrspace(5) %alloca, i32 0, i32 15
+ store i32 42, ptr addrspace(5) %gep.0
+ store i32 43, ptr addrspace(5) %gep.1
+ %gep = getelementptr inbounds [16 x i32], ptr addrspace(5) %alloca, i32 0, i32 %sel2
+ %load = load i32, ptr addrspace(5) %gep
+ store i32 %load, ptr %out
+ ret void
+}
+
declare i32 @llvm.amdgcn.workitem.id.x()
declare i32 @llvm.amdgcn.workitem.id.y()
declare void @llvm.memset.p5.i32(ptr addrspace(5) nocapture writeonly, i8, i32, i1 immarg)
@@ -278,5 +316,4 @@ declare void @llvm.memset.p5.i32(ptr addrspace(5) nocapture writeonly, i8, i32,
attributes #0 = { nounwind "amdgpu-promote-alloca-to-vector-max-regs"="24" "amdgpu-waves-per-eu"="4,4" }
attributes #1 = { nounwind "amdgpu-promote-alloca-to-vector-max-regs"="24" "amdgpu-waves-per-eu"="4,4" "amdgpu-promote-alloca-to-vector-vgpr-ratio"="2" }
attributes #2 = { nounwind "amdgpu-promote-alloca-to-vector-max-regs"="24" "amdgpu-waves-per-eu"="4,4" "amdgpu-promote-alloca-to-vector-vgpr-ratio"="8" }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; BASE: {{.*}}
+attributes #3 = { nounwind "amdgpu-promote-alloca-to-vector-max-regs"="24" "amdgpu-dynamic-vgpr-block-size"="16" }
>From 9b318abe153a4aa3d4749f0cea7585e0a119bb19 Mon Sep 17 00:00:00 2001
From: Mirko Brkusanin <Mirko.Brkusanin at amd.com>
Date: Fri, 25 Sep 2026 18:52:59 +0200
Subject: [PATCH 2/2] updates
---
llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp | 8 ++-
.../AMDGPU/dynamic-vgpr-entry-vgpr-limit.ll | 21 ++++--
.../AMDGPU/promote-alloca-vgpr-ratio.ll | 64 +++++++++++++++++++
3 files changed, 85 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index 26787724ac23c..b24f62fa69a11 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -1410,8 +1410,10 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
STM.getAddressableLocalMemorySize(), DS_Error));
}
- // Catches the paths the register allocator budget cannot constrain: explicit
- // physical registers in inline asm and wave dispatch VGPR arguments.
+ // When dynamic VGPRs are enabled, entry functions are launched with a single
+ // VGPR block. The register allocator enforces this constraint, but we also
+ // need to catch explicit physical registers in inline asm and wave dispatch
+ // VGPR arguments.
if (MFI->isDynamicVGPREnabled() &&
AMDGPU::isEntryFunctionCC(F.getCallingConv())) {
unsigned BlockSize = MFI->getDynamicVGPRBlockSize();
@@ -1421,7 +1423,7 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
LLVMContext &Ctx = F.getContext();
Ctx.diagnose(DiagnosticInfoResourceLimit(
F, "dynamic VGPR entry point vector registers", NumVgpr, BlockSize,
- DS_Error, DK_ResourceLimit));
+ DS_Warning, DK_ResourceLimit));
}
}
diff --git a/llvm/test/CodeGen/AMDGPU/dynamic-vgpr-entry-vgpr-limit.ll b/llvm/test/CodeGen/AMDGPU/dynamic-vgpr-entry-vgpr-limit.ll
index 8858d9c7ebf43..b8402f21bb078 100644
--- a/llvm/test/CodeGen/AMDGPU/dynamic-vgpr-entry-vgpr-limit.ll
+++ b/llvm/test/CodeGen/AMDGPU/dynamic-vgpr-entry-vgpr-limit.ll
@@ -1,28 +1,39 @@
-; RUN: not llc -mtriple=amdgpu12.00-amd-amdpal -filetype=null < %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple=amdgpu12.00-amd-amdpal -filetype=null < %s 2>&1 | FileCheck %s
-; CHECK: error: {{.*}}dynamic VGPR entry point vector registers (21) exceeds limit (16) in function 'entry_block16'
+; CHECK: warning: {{.*}}dynamic VGPR entry point vector registers (21) exceeds limit (16) in function 'entry_block16'
define amdgpu_cs void @entry_block16() #0 {
call void asm sideeffect "", "~{v20}"()
ret void
}
-; CHECK: error: {{.*}}dynamic VGPR entry point vector registers (21) exceeds limit (16) in function 'kernel_block16'
+; CHECK: warning: {{.*}}dynamic VGPR entry point vector registers (21) exceeds limit (16) in function 'kernel_block16'
define amdgpu_kernel void @kernel_block16() #0 {
call void asm sideeffect "", "~{v20}"()
ret void
}
-; CHECK-NOT: error{{.*}}'entry_block32'
+; CHECK-NOT: warning{{.*}}'entry_block32'
define amdgpu_cs void @entry_block32() #1 {
call void asm sideeffect "", "~{v20}"()
ret void
}
-; CHECK-NOT: error{{.*}}'chain_block16'
+; CHECK-NOT: warning{{.*}}'chain_block16'
define amdgpu_cs_chain void @chain_block16() #0 {
call void asm sideeffect "", "~{v20}"()
unreachable
}
+define amdgpu_gfx void @gfx_func_high_pressure() #0 {
+ call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7},~{v8},~{v9},~{v10},~{v11},~{v12},~{v13},~{v14},~{v15},~{v16},~{v17},~{v18},~{v19},~{v20},~{v21},~{v22},~{v23},~{v24},~{v25},~{v26},~{v27},~{v28},~{v29},~{v30},~{v31},~{v32},~{v33},~{v34},~{v35},~{v36},~{v37},~{v38},~{v39}"()
+ ret void
+}
+
+; CHECK: warning: {{.*}}dynamic VGPR entry point vector registers (40) exceeds limit (16) in function 'entry_calls_high_pressure'
+define amdgpu_cs void @entry_calls_high_pressure() #0 {
+ call amdgpu_gfx void @gfx_func_high_pressure()
+ ret void
+}
+
attributes #0 = { nounwind "amdgpu-dynamic-vgpr-block-size"="16" }
attributes #1 = { nounwind "amdgpu-dynamic-vgpr-block-size"="32" }
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-vgpr-ratio.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-vgpr-ratio.ll
index 5e767db0d7b19..93c8369dd74c2 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-vgpr-ratio.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-vgpr-ratio.ll
@@ -309,6 +309,70 @@ define amdgpu_cs void @i32_16_elements_dvgpr(ptr %out) #3 {
ret void
}
+define amdgpu_cs void @i32_4_elements_dvgpr(ptr %out) #3 {
+; DEFAULT-LABEL: define amdgpu_cs void @i32_4_elements_dvgpr(
+; DEFAULT-SAME: ptr [[OUT:%.*]]) #[[ATTR3]] {
+; DEFAULT-NEXT: [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; DEFAULT-NEXT: [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; DEFAULT-NEXT: [[C1:%.*]] = icmp uge i32 [[X]], 3
+; DEFAULT-NEXT: [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; DEFAULT-NEXT: [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; DEFAULT-NEXT: [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; DEFAULT-NEXT: [[ALLOCA:%.*]] = freeze <4 x i32> poison
+; DEFAULT-NEXT: [[TMP1:%.*]] = extractelement <4 x i32> <i32 42, i32 0, i32 0, i32 43>, i32 [[SEL2]]
+; DEFAULT-NEXT: store i32 [[TMP1]], ptr [[OUT]], align 4
+; DEFAULT-NEXT: ret void
+;
+; RATIO2-LABEL: define amdgpu_cs void @i32_4_elements_dvgpr(
+; RATIO2-SAME: ptr [[OUT:%.*]]) #[[ATTR3]] {
+; RATIO2-NEXT: [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; RATIO2-NEXT: [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; RATIO2-NEXT: [[C1:%.*]] = icmp uge i32 [[X]], 3
+; RATIO2-NEXT: [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; RATIO2-NEXT: [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; RATIO2-NEXT: [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; RATIO2-NEXT: [[ALLOCA:%.*]] = freeze <4 x i32> poison
+; RATIO2-NEXT: [[TMP1:%.*]] = extractelement <4 x i32> <i32 42, i32 0, i32 0, i32 43>, i32 [[SEL2]]
+; RATIO2-NEXT: store i32 [[TMP1]], ptr [[OUT]], align 4
+; RATIO2-NEXT: ret void
+;
+; RATIO8-LABEL: define amdgpu_cs void @i32_4_elements_dvgpr(
+; RATIO8-SAME: ptr [[OUT:%.*]]) #[[ATTR3]] {
+; RATIO8-NEXT: [[X:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.x()
+; RATIO8-NEXT: [[Y:%.*]] = tail call i32 @llvm.amdgcn.workitem.id.y()
+; RATIO8-NEXT: [[C1:%.*]] = icmp uge i32 [[X]], 3
+; RATIO8-NEXT: [[C2:%.*]] = icmp uge i32 [[Y]], 3
+; RATIO8-NEXT: [[SEL1:%.*]] = select i1 [[C1]], i32 1, i32 2
+; RATIO8-NEXT: [[SEL2:%.*]] = select i1 [[C2]], i32 0, i32 [[SEL1]]
+; RATIO8-NEXT: [[ALLOCA:%.*]] = alloca [4 x i32], align 16, addrspace(5)
+; RATIO8-NEXT: call void @llvm.memset.p5.i32(ptr addrspace(5) [[ALLOCA]], i8 0, i32 16, i1 false)
+; RATIO8-NEXT: [[GEP_0:%.*]] = getelementptr inbounds [4 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 0
+; RATIO8-NEXT: [[GEP_1:%.*]] = getelementptr inbounds [4 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 3
+; RATIO8-NEXT: store i32 42, ptr addrspace(5) [[GEP_0]], align 4
+; RATIO8-NEXT: store i32 43, ptr addrspace(5) [[GEP_1]], align 4
+; RATIO8-NEXT: [[GEP:%.*]] = getelementptr inbounds [4 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[SEL2]]
+; RATIO8-NEXT: [[LOAD:%.*]] = load i32, ptr addrspace(5) [[GEP]], align 4
+; RATIO8-NEXT: store i32 [[LOAD]], ptr [[OUT]], align 4
+; RATIO8-NEXT: ret void
+;
+ %x = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %y = tail call i32 @llvm.amdgcn.workitem.id.y()
+ %c1 = icmp uge i32 %x, 3
+ %c2 = icmp uge i32 %y, 3
+ %sel1 = select i1 %c1, i32 1, i32 2
+ %sel2 = select i1 %c2, i32 0, i32 %sel1
+ %alloca = alloca [4 x i32], align 16, addrspace(5)
+ call void @llvm.memset.p5.i32(ptr addrspace(5) %alloca, i8 0, i32 16, i1 false)
+ %gep.0 = getelementptr inbounds [4 x i32], ptr addrspace(5) %alloca, i32 0, i32 0
+ %gep.1 = getelementptr inbounds [4 x i32], ptr addrspace(5) %alloca, i32 0, i32 3
+ store i32 42, ptr addrspace(5) %gep.0
+ store i32 43, ptr addrspace(5) %gep.1
+ %gep = getelementptr inbounds [4 x i32], ptr addrspace(5) %alloca, i32 0, i32 %sel2
+ %load = load i32, ptr addrspace(5) %gep
+ store i32 %load, ptr %out
+ ret void
+}
+
declare i32 @llvm.amdgcn.workitem.id.x()
declare i32 @llvm.amdgcn.workitem.id.y()
declare void @llvm.memset.p5.i32(ptr addrspace(5) nocapture writeonly, i8, i32, i1 immarg)
More information about the llvm-commits
mailing list