[llvm] [AMDGPU] Multi dword spilling for unaligned tuples (PR #183701)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 10 02:04:57 PDT 2026
https://github.com/easyonaadit updated https://github.com/llvm/llvm-project/pull/183701
>From 100602e76b5825464dd97d063bd2c1ea97a8d798 Mon Sep 17 00:00:00 2001
From: Aaditya <Aaditya.AlokDeshpande at amd.com>
Date: Thu, 22 Jan 2026 10:43:49 +0530
Subject: [PATCH 1/7] [AMDGPU] Multi dword spilling for unaligned tuples
While spilling unaligned tuples, rather than breaking the
spill into 32-bit accesses, spill the first register as a
single 32-bit spill, and spill the remainder of the tuple
as an aligned tuple.
Some additional bookkeeping is required in the spilling
loop to manage the state.
---
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 51 +++++++++++++++++++----
llvm/test/CodeGen/AMDGPU/vgpr-spill.mir | 34 ++++-----------
2 files changed, 52 insertions(+), 33 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index e00ce4f167c3f..74ff4d90d6df5 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -1537,9 +1537,13 @@ void SIRegisterInfo::buildSpillLoadStore(
const unsigned RegWidth = AMDGPU::getRegBitWidth(*RC) / 8;
// On targets with register tuple alignment requirements,
- // for unaligned tuples, break the spill into 32-bit pieces.
- // TODO: Optimize misaligned spills by using larger aligned chunks instead of
- // 32-bit splits.
+ // for unaligned tuples, spill the first sub-reg as a 32-bit spill,
+ // and spill the rest as a regular aligned tuple.
+ // eg: SPILL_V224 $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+ // will be spilt as:
+ // SPILL_SCRATCH_DWORD $vgpr1
+ // SPILL_SCRATCH_DWORDx4 $vgpr2_vgpr3_vgpr4_vgpr5
+ // SPILL_SCRATCH_DWORDx2 $vgpr6_vgpr7
bool IsRegMisaligned = false;
if (!IsBlock && RegWidth > 4) {
unsigned SpillOpcode =
@@ -1561,14 +1565,24 @@ void SIRegisterInfo::buildSpillLoadStore(
// Always use 4 byte operations for AGPRs because we need to scavenge
// a temporary VGPR.
// If we're using a block operation, the element should be the whole block.
- // For misaligned registers, use 4-byte elements to avoid alignment errors.
+ // clang-format off
unsigned EltSize = IsBlock ? RegWidth
- : (IsFlat && !IsAGPR && !IsRegMisaligned)
+ : (IsFlat && !IsAGPR)
? std::min(RegWidth, 16u)
: 4u;
+ // clang-format on
unsigned NumSubRegs = RegWidth / EltSize;
unsigned Size = NumSubRegs * EltSize;
unsigned RemSize = RegWidth - Size;
+ // For unaligned tuples, the first sub-reg is spilt as a single 32-bit spill,
+ // and will count as an additional reg, so the last chunk will have one less
+ // register. In some cases, the last chunk could be completly eliminated,
+ // eg: SPILL_V160 $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 will be spilt as:
+ // SPILL_SCRATCH_DWORD $vgpr1
+ // SPILL_SCRATCH_DWORD $vgpr2_vgpr3_vgpr4_vgpr5
+ unsigned LastChunk = ((RemSize / 4) + 3) % 4;
+ if (IsRegMisaligned && LastChunk)
+ NumSubRegs += 1;
unsigned NumRemSubRegs = RemSize ? 1 : 0;
int64_t Offset = InstOffset + MFI.getObjectOffset(Index);
int64_t MaterializedOffset = Offset;
@@ -1732,10 +1746,31 @@ void SIRegisterInfo::buildSpillLoadStore(
for (unsigned i = 0, e = NumSubRegs + NumRemSubRegs, RegOffset = 0; i != e;
++i, RegOffset += EltSize) {
- if (i == NumSubRegs) {
- EltSize = RemSize;
+ unsigned SavedEltSize = EltSize;
+ if (i == 0 && IsRegMisaligned) {
+ // For misaligned register tuples, spill only the first sub-reg in the
+ // first iteration.
+ EltSize = 4u;
LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
}
+ if (i == 1 && IsRegMisaligned) {
+ // The first sub-reg was split in the previous iteration.
+ RegOffset = 4u;
+ if (RegWidth <= 16)
+ EltSize = RegWidth - 4u;
+ LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
+ }
+ if (IsRegMisaligned) {
+ if (i == (e - 1)) {
+ EltSize = LastChunk * 4;
+ LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
+ }
+ } else {
+ if (i == NumSubRegs) {
+ EltSize = RemSize;
+ LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
+ }
+ }
Desc = &TII->get(LoadStoreOp);
if (!IsFlat && UseVGPROffset) {
@@ -1969,6 +2004,8 @@ void SIRegisterInfo::buildSpillLoadStore(
// scavenged.
if (!IsStore && TII->isBlockLoadStore(LoadStoreOp))
addImplicitUsesForBlockCSRLoad(MIB, ValueReg);
+ if (i == 0 && IsRegMisaligned)
+ EltSize = SavedEltSize;
}
if (ScratchOffsetRegDelta != 0) {
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-spill.mir b/llvm/test/CodeGen/AMDGPU/vgpr-spill.mir
index ea1f68b8b6573..da02ca49940d0 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-spill.mir
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-spill.mir
@@ -290,9 +290,7 @@ body: |
; GFX942: liveins: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX942-NEXT: {{ $}}
; GFX942-NEXT: SCRATCH_STORE_DWORD_SADDR killed $vgpr1, $sgpr32, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr1_vgpr2_vgpr3_vgpr4, implicit $vgpr1_vgpr2_vgpr3_vgpr4 :: ("amdgpu-thread-private" store (s32) into %stack.0, addrspace 5)
- ; GFX942-NEXT: SCRATCH_STORE_DWORD_SADDR killed $vgpr2, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s32) into %stack.0 + 4, addrspace 5)
- ; GFX942-NEXT: SCRATCH_STORE_DWORD_SADDR killed $vgpr3, $sgpr32, 8, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s32) into %stack.0 + 8, addrspace 5)
- ; GFX942-NEXT: SCRATCH_STORE_DWORD_SADDR killed $vgpr4, $sgpr32, 12, 0, implicit $exec, implicit $flat_scr, implicit killed $vgpr1_vgpr2_vgpr3_vgpr4 :: ("amdgpu-thread-private" store (s32) into %stack.0 + 12, addrspace 5)
+ ; GFX942-NEXT: SCRATCH_STORE_DWORDX3_SADDR killed $vgpr2_vgpr3_vgpr4, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr, implicit killed $vgpr1_vgpr2_vgpr3_vgpr4 :: ("amdgpu-thread-private" store (s96) into %stack.0 + 4, align 4, addrspace 5)
;
; GFX1200-LABEL: name: spill_v128_kill_unaligned
; GFX1200: liveins: $vgpr1_vgpr2_vgpr3_vgpr4
@@ -303,9 +301,7 @@ body: |
; GFX1250: liveins: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: SCRATCH_STORE_DWORD_SADDR killed $vgpr1, $sgpr32, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr1_vgpr2_vgpr3_vgpr4, implicit $vgpr1_vgpr2_vgpr3_vgpr4 :: ("amdgpu-thread-private" store (s32) into %stack.0, addrspace 5)
- ; GFX1250-NEXT: SCRATCH_STORE_DWORD_SADDR killed $vgpr2, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s32) into %stack.0 + 4, addrspace 5)
- ; GFX1250-NEXT: SCRATCH_STORE_DWORD_SADDR killed $vgpr3, $sgpr32, 8, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s32) into %stack.0 + 8, addrspace 5)
- ; GFX1250-NEXT: SCRATCH_STORE_DWORD_SADDR killed $vgpr4, $sgpr32, 12, 0, implicit $exec, implicit $flat_scr, implicit killed $vgpr1_vgpr2_vgpr3_vgpr4 :: ("amdgpu-thread-private" store (s32) into %stack.0 + 12, addrspace 5)
+ ; GFX1250-NEXT: SCRATCH_STORE_DWORDX3_SADDR killed $vgpr2_vgpr3_vgpr4, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr, implicit killed $vgpr1_vgpr2_vgpr3_vgpr4 :: ("amdgpu-thread-private" store (s96) into %stack.0 + 4, align 4, addrspace 5)
SI_SPILL_V128_SAVE killed $vgpr1_vgpr2_vgpr3_vgpr4, %stack.0, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.0, addrspace 5)
...
@@ -334,9 +330,7 @@ body: |
; GFX942: liveins: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX942-NEXT: {{ $}}
; GFX942-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr1, $sgpr32, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr1_vgpr2_vgpr3_vgpr4, implicit $vgpr1_vgpr2_vgpr3_vgpr4 :: ("amdgpu-thread-private" store (s32) into %stack.0, addrspace 5)
- ; GFX942-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr2, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s32) into %stack.0 + 4, addrspace 5)
- ; GFX942-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr3, $sgpr32, 8, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s32) into %stack.0 + 8, addrspace 5)
- ; GFX942-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr4, $sgpr32, 12, 0, implicit $exec, implicit $flat_scr, implicit $vgpr1_vgpr2_vgpr3_vgpr4 :: ("amdgpu-thread-private" store (s32) into %stack.0 + 12, addrspace 5)
+ ; GFX942-NEXT: SCRATCH_STORE_DWORDX3_SADDR $vgpr2_vgpr3_vgpr4, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr, implicit $vgpr1_vgpr2_vgpr3_vgpr4 :: ("amdgpu-thread-private" store (s96) into %stack.0 + 4, align 4, addrspace 5)
;
; GFX1200-LABEL: name: spill_v128_unaligned
; GFX1200: liveins: $vgpr1_vgpr2_vgpr3_vgpr4
@@ -347,9 +341,7 @@ body: |
; GFX1250: liveins: $vgpr1_vgpr2_vgpr3_vgpr4
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr1, $sgpr32, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr1_vgpr2_vgpr3_vgpr4, implicit $vgpr1_vgpr2_vgpr3_vgpr4 :: ("amdgpu-thread-private" store (s32) into %stack.0, addrspace 5)
- ; GFX1250-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr2, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s32) into %stack.0 + 4, addrspace 5)
- ; GFX1250-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr3, $sgpr32, 8, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s32) into %stack.0 + 8, addrspace 5)
- ; GFX1250-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr4, $sgpr32, 12, 0, implicit $exec, implicit $flat_scr, implicit $vgpr1_vgpr2_vgpr3_vgpr4 :: ("amdgpu-thread-private" store (s32) into %stack.0 + 12, addrspace 5)
+ ; GFX1250-NEXT: SCRATCH_STORE_DWORDX3_SADDR $vgpr2_vgpr3_vgpr4, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr, implicit $vgpr1_vgpr2_vgpr3_vgpr4 :: ("amdgpu-thread-private" store (s96) into %stack.0 + 4, align 4, addrspace 5)
SI_SPILL_V128_SAVE $vgpr1_vgpr2_vgpr3_vgpr4, %stack.0, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.0, addrspace 5)
...
@@ -421,13 +413,8 @@ body: |
; GFX942: liveins: $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8
; GFX942-NEXT: {{ $}}
; GFX942-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr1, $sgpr32, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8, implicit $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8 :: ("amdgpu-thread-private" store (s32) into %stack.0, addrspace 5)
- ; GFX942-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr2, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s32) into %stack.0 + 4, addrspace 5)
- ; GFX942-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr3, $sgpr32, 8, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s32) into %stack.0 + 8, addrspace 5)
- ; GFX942-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr4, $sgpr32, 12, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s32) into %stack.0 + 12, addrspace 5)
- ; GFX942-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr5, $sgpr32, 16, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s32) into %stack.0 + 16, addrspace 5)
- ; GFX942-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr6, $sgpr32, 20, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s32) into %stack.0 + 20, addrspace 5)
- ; GFX942-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr7, $sgpr32, 24, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s32) into %stack.0 + 24, addrspace 5)
- ; GFX942-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr8, $sgpr32, 28, 0, implicit $exec, implicit $flat_scr, implicit $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8 :: ("amdgpu-thread-private" store (s32) into %stack.0 + 28, addrspace 5)
+ ; GFX942-NEXT: SCRATCH_STORE_DWORDX4_SADDR $vgpr2_vgpr3_vgpr4_vgpr5, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s128) into %stack.0 + 4, align 4, addrspace 5)
+ ; GFX942-NEXT: SCRATCH_STORE_DWORDX3_SADDR $vgpr6_vgpr7_vgpr8, $sgpr32, 20, 0, implicit $exec, implicit $flat_scr, implicit $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8 :: ("amdgpu-thread-private" store (s96) into %stack.0 + 20, align 4, addrspace 5)
;
; GFX1200-LABEL: name: spill_v256_unaligned
; GFX1200: liveins: $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8
@@ -439,12 +426,7 @@ body: |
; GFX1250: liveins: $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr1, $sgpr32, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8, implicit $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8 :: ("amdgpu-thread-private" store (s32) into %stack.0, addrspace 5)
- ; GFX1250-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr2, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s32) into %stack.0 + 4, addrspace 5)
- ; GFX1250-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr3, $sgpr32, 8, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s32) into %stack.0 + 8, addrspace 5)
- ; GFX1250-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr4, $sgpr32, 12, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s32) into %stack.0 + 12, addrspace 5)
- ; GFX1250-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr5, $sgpr32, 16, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s32) into %stack.0 + 16, addrspace 5)
- ; GFX1250-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr6, $sgpr32, 20, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s32) into %stack.0 + 20, addrspace 5)
- ; GFX1250-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr7, $sgpr32, 24, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s32) into %stack.0 + 24, addrspace 5)
- ; GFX1250-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr8, $sgpr32, 28, 0, implicit $exec, implicit $flat_scr, implicit $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8 :: ("amdgpu-thread-private" store (s32) into %stack.0 + 28, addrspace 5)
+ ; GFX1250-NEXT: SCRATCH_STORE_DWORDX4_SADDR $vgpr2_vgpr3_vgpr4_vgpr5, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s128) into %stack.0 + 4, align 4, addrspace 5)
+ ; GFX1250-NEXT: SCRATCH_STORE_DWORDX3_SADDR $vgpr6_vgpr7_vgpr8, $sgpr32, 20, 0, implicit $exec, implicit $flat_scr, implicit $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8 :: ("amdgpu-thread-private" store (s96) into %stack.0 + 20, align 4, addrspace 5)
SI_SPILL_V256_SAVE $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8, %stack.0, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.0, addrspace 5)
...
>From 1889bea19e5b4e5ed4d0ba5375047683bb60659a Mon Sep 17 00:00:00 2001
From: Aaditya <Aaditya.AlokDeshpande at amd.com>
Date: Mon, 9 Mar 2026 12:01:21 +0530
Subject: [PATCH 2/7] Exclude AGPRs from the alignment check.
AGPR spills are lowered to 4-byte access by default,
as we need to scavenge a temporary vgpr for them.
---
llvm/lib/Target/AMDGPU/CMakeLists.txt | 7 +++++++
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 2 +-
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index ae684a58cfd26..b57de19c75a54 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -2,6 +2,13 @@ add_llvm_component_group(AMDGPU)
set(LLVM_TARGET_DEFINITIONS AMDGPU.td)
+set_source_files_properties(
+ SIRegisterInfo.cpp
+ SIISelLowering.cpp
+ SIWholeQuadMode.cpp
+ PROPERTIES COMPILE_FLAGS "-O0 -g"
+)
+
tablegen(LLVM AMDGPUGenAsmMatcher.inc -gen-asm-matcher)
tablegen(LLVM AMDGPUGenAsmWriter.inc -gen-asm-writer)
tablegen(LLVM AMDGPUGenCallingConv.inc -gen-callingconv)
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 74ff4d90d6df5..920c75e3b2049 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -1545,7 +1545,7 @@ void SIRegisterInfo::buildSpillLoadStore(
// SPILL_SCRATCH_DWORDx4 $vgpr2_vgpr3_vgpr4_vgpr5
// SPILL_SCRATCH_DWORDx2 $vgpr6_vgpr7
bool IsRegMisaligned = false;
- if (!IsBlock && RegWidth > 4) {
+ if (!IsBlock && !IsAGPR && RegWidth > 4) {
unsigned SpillOpcode =
getFlatScratchSpillOpcode(TII, LoadStoreOp, std::min(RegWidth, 16u));
int VDataIdx =
>From 6c2d66660a58ab9556b11dc1718a507033629c33 Mon Sep 17 00:00:00 2001
From: Aaditya <Aaditya.AlokDeshpande at amd.com>
Date: Mon, 9 Mar 2026 13:39:00 +0530
Subject: [PATCH 3/7] Code cleanup
---
llvm/lib/Target/AMDGPU/CMakeLists.txt | 7 -------
1 file changed, 7 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index b57de19c75a54..ae684a58cfd26 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -2,13 +2,6 @@ add_llvm_component_group(AMDGPU)
set(LLVM_TARGET_DEFINITIONS AMDGPU.td)
-set_source_files_properties(
- SIRegisterInfo.cpp
- SIISelLowering.cpp
- SIWholeQuadMode.cpp
- PROPERTIES COMPILE_FLAGS "-O0 -g"
-)
-
tablegen(LLVM AMDGPUGenAsmMatcher.inc -gen-asm-matcher)
tablegen(LLVM AMDGPUGenAsmWriter.inc -gen-asm-writer)
tablegen(LLVM AMDGPUGenCallingConv.inc -gen-callingconv)
>From c71bf5c8e752b6e79ba763654f46ac79d71513fc Mon Sep 17 00:00:00 2001
From: Aaditya <Aaditya.AlokDeshpande at amd.com>
Date: Wed, 11 Mar 2026 13:54:03 +0530
Subject: [PATCH 4/7] Simplify spilling loop.
Plus some code cleanup, enable clang formatting.
---
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 37 ++++++++---------------
1 file changed, 13 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 920c75e3b2049..eed4c8bf56051 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -1565,17 +1565,14 @@ void SIRegisterInfo::buildSpillLoadStore(
// Always use 4 byte operations for AGPRs because we need to scavenge
// a temporary VGPR.
// If we're using a block operation, the element should be the whole block.
- // clang-format off
- unsigned EltSize = IsBlock ? RegWidth
- : (IsFlat && !IsAGPR)
- ? std::min(RegWidth, 16u)
- : 4u;
- // clang-format on
+ unsigned EltSize = IsBlock ? RegWidth
+ : (IsFlat && !IsAGPR) ? std::min(RegWidth, 16u)
+ : 4u;
unsigned NumSubRegs = RegWidth / EltSize;
unsigned Size = NumSubRegs * EltSize;
unsigned RemSize = RegWidth - Size;
// For unaligned tuples, the first sub-reg is spilt as a single 32-bit spill,
- // and will count as an additional reg, so the last chunk will have one less
+ // and will count as an additional reg, so the last chunk will have one less
// register. In some cases, the last chunk could be completly eliminated,
// eg: SPILL_V160 $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 will be spilt as:
// SPILL_SCRATCH_DWORD $vgpr1
@@ -1744,9 +1741,9 @@ void SIRegisterInfo::buildSpillLoadStore(
Desc = &TII->get(LoadStoreOp);
}
+ unsigned SavedEltSize = EltSize;
for (unsigned i = 0, e = NumSubRegs + NumRemSubRegs, RegOffset = 0; i != e;
++i, RegOffset += EltSize) {
- unsigned SavedEltSize = EltSize;
if (i == 0 && IsRegMisaligned) {
// For misaligned register tuples, spill only the first sub-reg in the
// first iteration.
@@ -1754,22 +1751,16 @@ void SIRegisterInfo::buildSpillLoadStore(
LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
}
if (i == 1 && IsRegMisaligned) {
- // The first sub-reg was split in the previous iteration.
- RegOffset = 4u;
- if (RegWidth <= 16)
- EltSize = RegWidth - 4u;
+ // The first sub-reg was spilt in the previous iteration.
+ EltSize = RegWidth <= 16 ? RegWidth - 4u : SavedEltSize;
LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
}
- if (IsRegMisaligned) {
- if (i == (e - 1)) {
- EltSize = LastChunk * 4;
- LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
- }
- } else {
- if (i == NumSubRegs) {
- EltSize = RemSize;
- LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
- }
+ if (IsRegMisaligned && i == (e - 1)) {
+ EltSize = LastChunk * 4;
+ LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
+ } else if (i == NumSubRegs) {
+ EltSize = RemSize;
+ LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
}
Desc = &TII->get(LoadStoreOp);
@@ -2004,8 +1995,6 @@ void SIRegisterInfo::buildSpillLoadStore(
// scavenged.
if (!IsStore && TII->isBlockLoadStore(LoadStoreOp))
addImplicitUsesForBlockCSRLoad(MIB, ValueReg);
- if (i == 0 && IsRegMisaligned)
- EltSize = SavedEltSize;
}
if (ScratchOffsetRegDelta != 0) {
>From c035183b4c73fd9b08da221a2c6fefb04f49fa48 Mon Sep 17 00:00:00 2001
From: Aaditya <Aaditya.AlokDeshpande at amd.com>
Date: Wed, 11 Mar 2026 14:04:55 +0530
Subject: [PATCH 5/7] further simplify loop logic.
---
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index eed4c8bf56051..19019947ef247 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -1578,7 +1578,7 @@ void SIRegisterInfo::buildSpillLoadStore(
// SPILL_SCRATCH_DWORD $vgpr1
// SPILL_SCRATCH_DWORD $vgpr2_vgpr3_vgpr4_vgpr5
unsigned LastChunk = ((RemSize / 4) + 3) % 4;
- if (IsRegMisaligned && LastChunk)
+ if (IsRegMisaligned && (LastChunk = (LastChunk * 4)))
NumSubRegs += 1;
unsigned NumRemSubRegs = RemSize ? 1 : 0;
int64_t Offset = InstOffset + MFI.getObjectOffset(Index);
@@ -1741,7 +1741,6 @@ void SIRegisterInfo::buildSpillLoadStore(
Desc = &TII->get(LoadStoreOp);
}
- unsigned SavedEltSize = EltSize;
for (unsigned i = 0, e = NumSubRegs + NumRemSubRegs, RegOffset = 0; i != e;
++i, RegOffset += EltSize) {
if (i == 0 && IsRegMisaligned) {
@@ -1752,11 +1751,11 @@ void SIRegisterInfo::buildSpillLoadStore(
}
if (i == 1 && IsRegMisaligned) {
// The first sub-reg was spilt in the previous iteration.
- EltSize = RegWidth <= 16 ? RegWidth - 4u : SavedEltSize;
+ EltSize = RegWidth <= 16 ? RegWidth - 4u : 16u;
LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
}
if (IsRegMisaligned && i == (e - 1)) {
- EltSize = LastChunk * 4;
+ EltSize = LastChunk;
LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
} else if (i == NumSubRegs) {
EltSize = RemSize;
>From 24a3a4ff225b6c1c80ed5a98c29b221d38e6519e Mon Sep 17 00:00:00 2001
From: Aaditya <Aaditya.AlokDeshpande at amd.com>
Date: Mon, 30 Mar 2026 11:07:10 +0530
Subject: [PATCH 6/7] simplify if statements
---
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 28 +++++++++++------------
llvm/test/CodeGen/AMDGPU/vgpr-spill.mir | 28 ++++++++++++-----------
2 files changed, 29 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 19019947ef247..ddb697ac3ec28 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -1743,20 +1743,20 @@ void SIRegisterInfo::buildSpillLoadStore(
for (unsigned i = 0, e = NumSubRegs + NumRemSubRegs, RegOffset = 0; i != e;
++i, RegOffset += EltSize) {
- if (i == 0 && IsRegMisaligned) {
- // For misaligned register tuples, spill only the first sub-reg in the
- // first iteration.
- EltSize = 4u;
- LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
- }
- if (i == 1 && IsRegMisaligned) {
- // The first sub-reg was spilt in the previous iteration.
- EltSize = RegWidth <= 16 ? RegWidth - 4u : 16u;
- LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
- }
- if (IsRegMisaligned && i == (e - 1)) {
- EltSize = LastChunk;
- LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
+ if (IsRegMisaligned) {
+ if (i == 0) {
+ // For misaligned register tuples, spill only the first sub-reg in the
+ // first iteration.
+ EltSize = 4u;
+ LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
+ } else if (i == 1) {
+ // The first sub-reg was spilt in the previous iteration.
+ EltSize = RegWidth <= 16 ? RegWidth - 4u : 16u;
+ LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
+ } else if (LastChunk && (i + 1) == e) {
+ EltSize = LastChunk;
+ LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
+ }
} else if (i == NumSubRegs) {
EltSize = RemSize;
LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-spill.mir b/llvm/test/CodeGen/AMDGPU/vgpr-spill.mir
index da02ca49940d0..95a8d5d87eead 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-spill.mir
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-spill.mir
@@ -316,33 +316,35 @@ machineFunctionInfo:
frameOffsetReg: '$sgpr33'
body: |
bb.0:
- liveins: $vgpr1_vgpr2_vgpr3_vgpr4
+ liveins: $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
; GFX900-LABEL: name: spill_v128_unaligned
- ; GFX900: liveins: $vgpr1_vgpr2_vgpr3_vgpr4
+ ; GFX900: liveins: $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
; GFX900-NEXT: {{ $}}
- ; GFX900-NEXT: BUFFER_STORE_DWORD_OFFSET $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, implicit $exec, implicit-def $vgpr1_vgpr2_vgpr3_vgpr4, implicit $vgpr1_vgpr2_vgpr3_vgpr4 :: ("amdgpu-thread-private" store (s32) into %stack.0, addrspace 5)
+ ; GFX900-NEXT: BUFFER_STORE_DWORD_OFFSET $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, implicit $exec, implicit-def $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5, implicit $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: ("amdgpu-thread-private" store (s32) into %stack.0, addrspace 5)
; GFX900-NEXT: BUFFER_STORE_DWORD_OFFSET $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.0 + 4, addrspace 5)
; GFX900-NEXT: BUFFER_STORE_DWORD_OFFSET $vgpr3, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 8, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.0 + 8, addrspace 5)
- ; GFX900-NEXT: BUFFER_STORE_DWORD_OFFSET $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, implicit $exec, implicit $vgpr1_vgpr2_vgpr3_vgpr4 :: ("amdgpu-thread-private" store (s32) into %stack.0 + 12, addrspace 5)
+ ; GFX900-NEXT: BUFFER_STORE_DWORD_OFFSET $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 12, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.0 + 12, addrspace 5)
+ ; GFX900-NEXT: BUFFER_STORE_DWORD_OFFSET $vgpr5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 16, 0, 0, implicit $exec, implicit $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: ("amdgpu-thread-private" store (s32) into %stack.0 + 16, addrspace 5)
;
; GFX942-LABEL: name: spill_v128_unaligned
- ; GFX942: liveins: $vgpr1_vgpr2_vgpr3_vgpr4
+ ; GFX942: liveins: $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr1, $sgpr32, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr1_vgpr2_vgpr3_vgpr4, implicit $vgpr1_vgpr2_vgpr3_vgpr4 :: ("amdgpu-thread-private" store (s32) into %stack.0, addrspace 5)
- ; GFX942-NEXT: SCRATCH_STORE_DWORDX3_SADDR $vgpr2_vgpr3_vgpr4, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr, implicit $vgpr1_vgpr2_vgpr3_vgpr4 :: ("amdgpu-thread-private" store (s96) into %stack.0 + 4, align 4, addrspace 5)
+ ; GFX942-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr1, $sgpr32, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5, implicit $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: ("amdgpu-thread-private" store (s32) into %stack.0, addrspace 5)
+ ; GFX942-NEXT: SCRATCH_STORE_DWORDX4_SADDR $vgpr2_vgpr3_vgpr4_vgpr5, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr, implicit $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: ("amdgpu-thread-private" store (s128) into %stack.0 + 4, align 4, addrspace 5)
;
; GFX1200-LABEL: name: spill_v128_unaligned
- ; GFX1200: liveins: $vgpr1_vgpr2_vgpr3_vgpr4
+ ; GFX1200: liveins: $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
; GFX1200-NEXT: {{ $}}
- ; GFX1200-NEXT: SCRATCH_STORE_DWORDX4_SADDR $vgpr1_vgpr2_vgpr3_vgpr4, $sgpr32, 0, 0, implicit $exec, implicit $flat_scr :: ("amdgpu-thread-private" store (s128) into %stack.0, align 4, addrspace 5)
+ ; GFX1200-NEXT: SCRATCH_STORE_DWORDX4_SADDR $vgpr1_vgpr2_vgpr3_vgpr4, $sgpr32, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5, implicit $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: ("amdgpu-thread-private" store (s128) into %stack.0, align 4, addrspace 5)
+ ; GFX1200-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr5, $sgpr32, 16, 0, implicit $exec, implicit $flat_scr, implicit $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: ("amdgpu-thread-private" store (s32) into %stack.0 + 16, addrspace 5)
;
; GFX1250-LABEL: name: spill_v128_unaligned
- ; GFX1250: liveins: $vgpr1_vgpr2_vgpr3_vgpr4
+ ; GFX1250: liveins: $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5
; GFX1250-NEXT: {{ $}}
- ; GFX1250-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr1, $sgpr32, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr1_vgpr2_vgpr3_vgpr4, implicit $vgpr1_vgpr2_vgpr3_vgpr4 :: ("amdgpu-thread-private" store (s32) into %stack.0, addrspace 5)
- ; GFX1250-NEXT: SCRATCH_STORE_DWORDX3_SADDR $vgpr2_vgpr3_vgpr4, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr, implicit $vgpr1_vgpr2_vgpr3_vgpr4 :: ("amdgpu-thread-private" store (s96) into %stack.0 + 4, align 4, addrspace 5)
- SI_SPILL_V128_SAVE $vgpr1_vgpr2_vgpr3_vgpr4, %stack.0, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.0, addrspace 5)
+ ; GFX1250-NEXT: SCRATCH_STORE_DWORD_SADDR $vgpr1, $sgpr32, 0, 0, implicit $exec, implicit $flat_scr, implicit-def $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5, implicit $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: ("amdgpu-thread-private" store (s32) into %stack.0, addrspace 5)
+ ; GFX1250-NEXT: SCRATCH_STORE_DWORDX4_SADDR $vgpr2_vgpr3_vgpr4_vgpr5, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr, implicit $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 :: ("amdgpu-thread-private" store (s128) into %stack.0 + 4, align 4, addrspace 5)
+ SI_SPILL_V160_SAVE $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5, %stack.0, $sgpr32, 0, implicit $exec :: (store (s128) into %stack.0, addrspace 5)
...
---
>From c2cbc14913c805094747c0723333c88a0ef53877 Mon Sep 17 00:00:00 2001
From: Aaditya <Aaditya.AlokDeshpande at amd.com>
Date: Fri, 10 Apr 2026 14:34:38 +0530
Subject: [PATCH 7/7] Refactor: Remove reduntant calls
---
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 13 +++++--------
1 file changed, 5 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index ddb697ac3ec28..62a0b5d0c0818 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -1577,8 +1577,8 @@ void SIRegisterInfo::buildSpillLoadStore(
// eg: SPILL_V160 $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5 will be spilt as:
// SPILL_SCRATCH_DWORD $vgpr1
// SPILL_SCRATCH_DWORD $vgpr2_vgpr3_vgpr4_vgpr5
- unsigned LastChunk = ((RemSize / 4) + 3) % 4;
- if (IsRegMisaligned && (LastChunk = (LastChunk * 4)))
+ unsigned LastChunk = (((RemSize / 4) + 3) % 4) * 4;
+ if (IsRegMisaligned && LastChunk)
NumSubRegs += 1;
unsigned NumRemSubRegs = RemSize ? 1 : 0;
int64_t Offset = InstOffset + MFI.getObjectOffset(Index);
@@ -1748,19 +1748,16 @@ void SIRegisterInfo::buildSpillLoadStore(
// For misaligned register tuples, spill only the first sub-reg in the
// first iteration.
EltSize = 4u;
- LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
} else if (i == 1) {
// The first sub-reg was spilt in the previous iteration.
EltSize = RegWidth <= 16 ? RegWidth - 4u : 16u;
- LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
- } else if (LastChunk && (i + 1) == e) {
+ } else if (LastChunk && (i + 1) == e)
EltSize = LastChunk;
- LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
- }
} else if (i == NumSubRegs) {
EltSize = RemSize;
- LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
}
+ if (i == NumSubRegs || IsRegMisaligned)
+ LoadStoreOp = getFlatScratchSpillOpcode(TII, LoadStoreOp, EltSize);
Desc = &TII->get(LoadStoreOp);
if (!IsFlat && UseVGPROffset) {
More information about the llvm-commits
mailing list