[llvm] [AMDGPU] Insert spill codes for the SGPRs used for EXEC copy (PR #79428)
Christudasan Devadasan via llvm-commits
llvm-commits at lists.llvm.org
Tue Jan 30 07:01:32 PST 2024
https://github.com/cdevadas updated https://github.com/llvm/llvm-project/pull/79428
>From 0a4e12df3c51cf0203536c08bfd13686731c534f Mon Sep 17 00:00:00 2001
From: Christudasan Devadasan <Christudasan.Devadasan at amd.com>
Date: Tue, 30 Jan 2024 17:22:49 +0530
Subject: [PATCH] [AMDGPU] Insert spill codes for the SGPRs used for EXEC copy
The SGPR registers used for preserving EXEC mask while lowering
the whole-wave register spills and copies should be preserved at
the prolog and epilog if they are in the CSR range. It isn't
happening when there is only wwm-copy lowered and there are no
wwm-spills. This patch addresses that problem.
---
llvm/lib/Target/AMDGPU/SIFrameLowering.cpp | 16 ++++++---
.../AMDGPU/preserve-wwm-copy-dst-reg.ll | 36 +++++++++++++------
.../AMDGPU/whole-wave-register-copy.ll | 8 +++--
3 files changed, 42 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
index 9d062eb156d5c..c439c78a4e8fc 100644
--- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -1504,9 +1504,11 @@ void SIFrameLowering::determinePrologEpilogSGPRSaves(
const TargetRegisterClass &RC = *TRI->getWaveMaskRegClass();
- if (NeedExecCopyReservedReg) {
- Register ReservedReg = MFI->getSGPRForEXECCopy();
- assert(ReservedReg && "Should have reserved an SGPR for EXEC copy.");
+ Register ReservedRegForExecCopy = MFI->getSGPRForEXECCopy();
+ if (NeedExecCopyReservedReg ||
+ (ReservedRegForExecCopy &&
+ MRI.isPhysRegUsed(ReservedRegForExecCopy, /*SkipRegMaskTest=*/true))) {
+ MRI.reserveReg(ReservedRegForExecCopy, TRI);
Register UnusedScratchReg = findUnusedRegister(MRI, LiveUnits, RC);
if (UnusedScratchReg) {
// If found any unused scratch SGPR, reserve the register itself for Exec
@@ -1515,11 +1517,15 @@ void SIFrameLowering::determinePrologEpilogSGPRSaves(
LiveUnits.addReg(UnusedScratchReg);
} else {
// Needs spill.
- assert(!MFI->hasPrologEpilogSGPRSpillEntry(ReservedReg) &&
+ assert(!MFI->hasPrologEpilogSGPRSpillEntry(ReservedRegForExecCopy) &&
"Re-reserving spill slot for EXEC copy register");
- getVGPRSpillLaneOrTempRegister(MF, LiveUnits, ReservedReg, RC,
+ getVGPRSpillLaneOrTempRegister(MF, LiveUnits, ReservedRegForExecCopy, RC,
/*IncludeScratchCopy=*/false);
}
+ } else if (ReservedRegForExecCopy) {
+ // Reset it at this point. There are no whole-wave copies and spills
+ // encountered.
+ MFI->setSGPRForEXECCopy(AMDGPU::NoRegister);
}
// hasFP only knows about stack objects that already exist. We're now
diff --git a/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll b/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll
index b63c8038f9c81..5d580f838e0e0 100644
--- a/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll
+++ b/llvm/test/CodeGen/AMDGPU/preserve-wwm-copy-dst-reg.ll
@@ -32,9 +32,11 @@ define void @preserve_wwm_copy_dstreg(ptr %parg0, ptr %parg1, ptr %parg2) #0 {
; GFX906-NEXT: v_writelane_b32 v2, s24, 5
; GFX906-NEXT: s_mov_b64 s[26:27], s[10:11]
; GFX906-NEXT: v_writelane_b32 v2, s26, 6
+; GFX906-NEXT: v_writelane_b32 v41, s34, 2
; GFX906-NEXT: v_writelane_b32 v2, s27, 7
+; GFX906-NEXT: v_writelane_b32 v41, s35, 3
; GFX906-NEXT: v_writelane_b32 v2, s8, 8
-; GFX906-NEXT: v_writelane_b32 v41, s16, 2
+; GFX906-NEXT: v_writelane_b32 v41, s16, 4
; GFX906-NEXT: v_writelane_b32 v2, s9, 9
; GFX906-NEXT: v_writelane_b32 v41, s30, 0
; GFX906-NEXT: v_writelane_b32 v2, s4, 10
@@ -338,7 +340,9 @@ define void @preserve_wwm_copy_dstreg(ptr %parg0, ptr %parg1, ptr %parg2) #0 {
; GFX906-NEXT: v_readlane_b32 s31, v41, 1
; GFX906-NEXT: v_readlane_b32 s30, v41, 0
; GFX906-NEXT: ; kill: killed $vgpr40
-; GFX906-NEXT: v_readlane_b32 s4, v41, 2
+; GFX906-NEXT: v_readlane_b32 s34, v41, 2
+; GFX906-NEXT: v_readlane_b32 s35, v41, 3
+; GFX906-NEXT: v_readlane_b32 s4, v41, 4
; GFX906-NEXT: s_waitcnt vmcnt(0)
; GFX906-NEXT: flat_store_dwordx4 v[0:1], v[30:33] offset:112
; GFX906-NEXT: s_waitcnt vmcnt(0)
@@ -379,23 +383,27 @@ define void @preserve_wwm_copy_dstreg(ptr %parg0, ptr %parg1, ptr %parg2) #0 {
; GFX908-NEXT: s_mov_b64 exec, -1
; GFX908-NEXT: buffer_store_dword v40, off, s[0:3], s33 offset:152 ; 4-byte Folded Spill
; GFX908-NEXT: s_mov_b64 exec, s[18:19]
-; GFX908-NEXT: v_mov_b32_e32 v3, s16
+; GFX908-NEXT: v_mov_b32_e32 v3, s34
; GFX908-NEXT: buffer_store_dword v3, off, s[0:3], s33 offset:160 ; 4-byte Folded Spill
+; GFX908-NEXT: v_mov_b32_e32 v3, s35
+; GFX908-NEXT: buffer_store_dword v3, off, s[0:3], s33 offset:164 ; 4-byte Folded Spill
+; GFX908-NEXT: v_mov_b32_e32 v3, s16
+; GFX908-NEXT: buffer_store_dword v3, off, s[0:3], s33 offset:168 ; 4-byte Folded Spill
; GFX908-NEXT: s_addk_i32 s32, 0x2c00
; GFX908-NEXT: s_mov_b64 s[16:17], exec
; GFX908-NEXT: s_mov_b64 exec, 1
-; GFX908-NEXT: buffer_store_dword v2, off, s[0:3], s33 offset:164
+; GFX908-NEXT: buffer_store_dword v2, off, s[0:3], s33 offset:172
; GFX908-NEXT: v_writelane_b32 v2, s30, 0
; GFX908-NEXT: buffer_store_dword v2, off, s[0:3], s33 ; 4-byte Folded Spill
-; GFX908-NEXT: buffer_load_dword v2, off, s[0:3], s33 offset:164
+; GFX908-NEXT: buffer_load_dword v2, off, s[0:3], s33 offset:172
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_mov_b64 exec, s[16:17]
; GFX908-NEXT: s_mov_b64 s[16:17], exec
; GFX908-NEXT: s_mov_b64 exec, 1
-; GFX908-NEXT: buffer_store_dword v2, off, s[0:3], s33 offset:164
+; GFX908-NEXT: buffer_store_dword v2, off, s[0:3], s33 offset:172
; GFX908-NEXT: v_writelane_b32 v2, s31, 0
; GFX908-NEXT: buffer_store_dword v2, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
-; GFX908-NEXT: buffer_load_dword v2, off, s[0:3], s33 offset:164
+; GFX908-NEXT: buffer_load_dword v2, off, s[0:3], s33 offset:172
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_mov_b64 exec, s[16:17]
; GFX908-NEXT: ; implicit-def: $vgpr2
@@ -729,25 +737,31 @@ define void @preserve_wwm_copy_dstreg(ptr %parg0, ptr %parg1, ptr %parg2) #0 {
; GFX908-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_mov_b64 exec, 1
-; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s33 offset:164
+; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s33 offset:172
; GFX908-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:4 ; 4-byte Folded Reload
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_readlane_b32 s31, v0, 0
-; GFX908-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:164
+; GFX908-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:172
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_mov_b64 exec, s[4:5]
; GFX908-NEXT: s_mov_b64 s[4:5], exec
; GFX908-NEXT: s_mov_b64 exec, 1
-; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s33 offset:164
+; GFX908-NEXT: buffer_store_dword v0, off, s[0:3], s33 offset:172
; GFX908-NEXT: buffer_load_dword v0, off, s[0:3], s33 ; 4-byte Folded Reload
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_readlane_b32 s30, v0, 0
-; GFX908-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:164
+; GFX908-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:172
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_mov_b64 exec, s[4:5]
; GFX908-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:160 ; 4-byte Folded Reload
; GFX908-NEXT: ; kill: killed $vgpr40
; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_readfirstlane_b32 s34, v0
+; GFX908-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:164 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: v_readfirstlane_b32 s35, v0
+; GFX908-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:168 ; 4-byte Folded Reload
+; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_readfirstlane_b32 s4, v0
; GFX908-NEXT: s_xor_saveexec_b64 s[6:7], -1
; GFX908-NEXT: buffer_load_dword v33, off, s[0:3], s33 offset:148 ; 4-byte Folded Reload
diff --git a/llvm/test/CodeGen/AMDGPU/whole-wave-register-copy.ll b/llvm/test/CodeGen/AMDGPU/whole-wave-register-copy.ll
index cfb1cffdb5096..f680bbdd05cdd 100644
--- a/llvm/test/CodeGen/AMDGPU/whole-wave-register-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/whole-wave-register-copy.ll
@@ -18,7 +18,9 @@ define void @vector_reg_liverange_split() #0 {
; GFX90A-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
; GFX90A-NEXT: buffer_store_dword a32, off, s[0:3], s33 offset:4 ; 4-byte Folded Spill
; GFX90A-NEXT: s_mov_b64 exec, s[18:19]
-; GFX90A-NEXT: v_writelane_b32 v40, s16, 2
+; GFX90A-NEXT: v_writelane_b32 v40, s28, 2
+; GFX90A-NEXT: v_writelane_b32 v40, s29, 3
+; GFX90A-NEXT: v_writelane_b32 v40, s16, 4
; GFX90A-NEXT: ; implicit-def: $vgpr0 : SGPR spill to VGPR lane
; GFX90A-NEXT: v_writelane_b32 v40, s30, 0
; GFX90A-NEXT: s_addk_i32 s32, 0x400
@@ -46,7 +48,9 @@ define void @vector_reg_liverange_split() #0 {
; GFX90A-NEXT: v_readlane_b32 s31, v40, 1
; GFX90A-NEXT: v_readlane_b32 s30, v40, 0
; GFX90A-NEXT: ; kill: killed $vgpr0
-; GFX90A-NEXT: v_readlane_b32 s4, v40, 2
+; GFX90A-NEXT: v_readlane_b32 s28, v40, 2
+; GFX90A-NEXT: v_readlane_b32 s29, v40, 3
+; GFX90A-NEXT: v_readlane_b32 s4, v40, 4
; GFX90A-NEXT: s_xor_saveexec_b64 s[6:7], -1
; GFX90A-NEXT: buffer_load_dword v0, off, s[0:3], s33 offset:8 ; 4-byte Folded Reload
; GFX90A-NEXT: s_mov_b64 exec, -1
More information about the llvm-commits
mailing list