[llvm-branch-commits] [llvm] [AMDGPU] Don't spill an SGPR while SCC is live in frame index lowering (PR #224080)
Domenic Nutile via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Sep 22 09:50:15 PDT 2026
https://github.com/saxlungs updated https://github.com/llvm/llvm-project/pull/224080
>From dd4ffa289bcf23b81e5085d5dcbf3eb68878ac92 Mon Sep 17 00:00:00 2001
From: Domenic Nutile <domenic.nutile at gmail.com>
Date: Wed, 16 Sep 2026 12:12:46 -0400
Subject: [PATCH 1/3] [AMDGPU] Don't spill an SGPR while SCC is live in frame
index lowering
When SCC is live into a scalar frame index user, the scaling path avoids
SALU ops that write SCC by computing the address in a VGPR and reading it
back with V_READFIRSTLANE_B32. If the destination of that readfirstlane is
scavenged with spilling allowed, an AMDGPU SGPR spill writes inactive
lanes, so it flips EXEC with S_NOT_B64 and clobbers SCC. Instead, scavenge
that register with AllowSpill=false.
---
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 39 ++++++--
...to-vmem-scc-clobber-reserved-exec-copy.mir | 98 +++++--------------
2 files changed, 55 insertions(+), 82 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 7cad384c98a8d..b6b1bccb0820d 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -3421,11 +3421,32 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
// Scaling FrameReg in place is the last resort when there is nothing to
// scavenge. It has to be undone after MI, which is only possible while MI
// does not use FrameReg for anything besides the frame index.
- bool CanUseFrameRegAsScratch = IsSALU && !LiveSCC && FrameReg &&
- !MI->readsRegister(FrameReg, this) &&
- !MI->modifiesRegister(FrameReg, this);
+ bool CanUseFrameRegAsSGPRScratch = IsSALU && FrameReg &&
+ !MI->readsRegister(FrameReg, this) &&
+ !MI->modifiesRegister(FrameReg, this);
+ // ResultReg is a VGPR while SCC is live, so FrameReg cannot stand in for
+ // it there.
+ bool CanUseFrameRegAsScratch = CanUseFrameRegAsSGPRScratch && !LiveSCC;
bool RestoreFrameReg = false;
+
+ // Scavenge the scalar temporary the V_READFIRSTLANE_B32 result lands in.
+ // Spilling an SGPR here would flip EXEC with S_NOT, and that clobbers the
+ // SCC this path exists to preserve, so fall back to FrameReg instead.
+ auto ScavengeSGPRForReadfirstlane =
+ [&](MachineBasicBlock::iterator To) -> Register {
+ if (Register Reg = RS->scavengeRegisterBackwards(
+ AMDGPU::SReg_32_XM0RegClass, To, false, 0,
+ /*AllowSpill=*/false))
+ return Reg;
+ if (CanUseFrameRegAsSGPRScratch) {
+ RestoreFrameReg = true;
+ return FrameReg;
+ }
+ return RS->scavengeRegisterBackwards(AMDGPU::SReg_32_XM0RegClass, To,
+ false, 0);
+ };
+
Register ResultReg;
if (IsCopy) {
ResultReg = MI->getOperand(0).getReg();
@@ -3479,8 +3500,7 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
assert(ResultReg.isPhysical());
NewDest = ResultReg;
} else {
- NewDest = RS->scavengeRegisterBackwards(AMDGPU::SReg_32_XM0RegClass,
- Shift, false, 0);
+ NewDest = ScavengeSGPRForReadfirstlane(Shift);
}
BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), NewDest)
.addReg(TmpResultReg);
@@ -3600,9 +3620,7 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
if (IsCopy) {
NewDest = ResultReg;
} else {
- NewDest = RS->scavengeRegisterBackwards(
- AMDGPU::SReg_32_XM0RegClass, *Add, false, 0,
- /*AllowSpill=*/true);
+ NewDest = ScavengeSGPRForReadfirstlane(*Add);
}
BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
@@ -3641,8 +3659,9 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
if (Offset) {
int64_t ScaledOffset = -Offset * ST.getWavefrontSize();
- bool SCCLiveAfterMI = MI->definesRegister(AMDGPU::SCC, this) &&
- !MI->registerDefIsDead(AMDGPU::SCC, this);
+ bool SCCLiveAfterMI = MI->definesRegister(AMDGPU::SCC, this)
+ ? !MI->registerDefIsDead(AMDGPU::SCC, this)
+ : LiveSCC;
if (!SCCLiveAfterMI) {
BuildMI(*MBB, InsPt, DL, TII->get(AMDGPU::S_ADD_I32), FrameReg)
.addReg(FrameReg)
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-to-vmem-scc-clobber-reserved-exec-copy.mir b/llvm/test/CodeGen/AMDGPU/sgpr-spill-to-vmem-scc-clobber-reserved-exec-copy.mir
index d9e1a0fcaf3a7..36a532d6c63e0 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-to-vmem-scc-clobber-reserved-exec-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-to-vmem-scc-clobber-reserved-exec-copy.mir
@@ -1,8 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: not llc -mtriple=amdgpu9.0a-amd-amdhsa -run-pass=prolog-epilog -o - %s 2>&1 | FileCheck %s
-
-# Lowering bails out on scc_live_into_scalar_frame_index_no_free_sgpr_nonzero_offset
-# CHECK-COUNT-4: error: {{.*}} in function scc_live_into_scalar_frame_index_no_free_sgpr_nonzero_offset void (): unhandled SGPR spill to memory
+# RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -run-pass=prolog-epilog -o - %s 2>&1 | FileCheck %s
---
name: scc_live_across_scalar_frame_index
tracksRegLiveness: true
@@ -173,9 +170,9 @@ body: |
---
# SCC is live *into* the scalar frame index user: S_CMP_EQ_U32 produces it and
# S_CSELECT_B32 reads it. With no SGPR free to hold the V_READFIRSTLANE_B32
-# result, scavenging one emergency-spills, and an SGPR spill flips EXEC with
-# S_NOT_B64. Those EXEC flips land between the S_CMP_EQ_U32 and the
-# S_CSELECT_B32 and clobber the SCC in between, which is wrong.
+# result, scavenging one would emergency-spill, and an SGPR spill flips EXEC
+# with S_NOT_B64, clobbering the very SCC this path preserves. Scale FrameReg in
+# place instead and restore it after MI with S_MUL_I32, which does not write SCC.
name: scc_live_into_scalar_frame_index_no_free_sgpr
tracksRegLiveness: true
stack:
@@ -202,27 +199,10 @@ body: |
; CHECK-NEXT: liveins: $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31_sgpr32_sgpr33_sgpr34_sgpr35, $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, $sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, $sgpr96_sgpr97_sgpr98_sgpr99, $vcc
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_CMP_EQ_U32 renamable $sgpr96, 0, implicit-def $scc
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc, implicit-def $vgpr1
- ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.2, addrspace 5)
- ; CHECK-NEXT: $vgpr1 = SI_SPILL_S32_TO_VGPR killed $sgpr4, 0, undef $vgpr1
- ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.2, addrspace 5)
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc
- ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.2, addrspace 5)
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc
- ; CHECK-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.2, addrspace 5)
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc, implicit killed $vgpr1
; CHECK-NEXT: $vgpr0 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
- ; CHECK-NEXT: $sgpr4 = V_READFIRSTLANE_B32 $vgpr0, implicit $exec
- ; CHECK-NEXT: renamable $vcc_lo = S_CSELECT_B32 killed $sgpr4, renamable $sgpr97, implicit $scc
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc, implicit-def $vgpr0
- ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.2, addrspace 5)
- ; CHECK-NEXT: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.2, addrspace 5)
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc
- ; CHECK-NEXT: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.2, addrspace 5)
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc
- ; CHECK-NEXT: $sgpr4 = SI_RESTORE_S32_FROM_VGPR killed $vgpr0, 0
- ; CHECK-NEXT: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.2, addrspace 5)
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc, implicit killed $vgpr0
+ ; CHECK-NEXT: $sgpr32 = V_READFIRSTLANE_B32 $vgpr0, implicit $exec
+ ; CHECK-NEXT: renamable $vcc_lo = S_CSELECT_B32 $sgpr32, renamable $sgpr97, implicit $scc
+ ; CHECK-NEXT: $sgpr32 = S_MUL_I32 $sgpr32, 64
; CHECK-NEXT: S_NOP 0, implicit killed $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, implicit killed $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31_sgpr32_sgpr33_sgpr34_sgpr35, implicit killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, implicit killed $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, implicit killed $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, implicit killed $sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, implicit killed $sgpr96_sgpr97_sgpr98_sgpr99
; CHECK-NEXT: S_BRANCH %bb.2
; CHECK-NEXT: {{ $}}
@@ -249,9 +229,10 @@ body: |
---
# As above, but with a non-zero offset and with SCC live *across* the frame index
-# user: S_CBRANCH_SCC1 still needs it after S_CSELECT_B32 has read it. Here the
-# scavenger cannot spill an SGPR at all without destroying SCC, so lowering bails
-# out with "unhandled SGPR spill to memory" and leaves both SCC reads undefined.
+# user: S_CBRANCH_SCC1 still needs it after S_CSELECT_B32 has read it. Restoring
+# FrameReg therefore has to fold the offset back without losing SCC, so S_MUL_I32
+# is followed by the S_ADDC_U32/S_BITCMP1_B32/S_BITSET0_B32 sequence that
+# smuggles SCC through bit 0, which the scaling has just cleared.
name: scc_live_into_scalar_frame_index_no_free_sgpr_nonzero_offset
tracksRegLiveness: true
stack:
@@ -279,27 +260,13 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_CMP_EQ_U32 renamable $sgpr96, 0, implicit-def $scc
; CHECK-NEXT: $vgpr0 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc, implicit-def $vgpr1
- ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.2, addrspace 5)
- ; CHECK-NEXT: $vgpr1 = SI_SPILL_S32_TO_VGPR killed $sgpr4, 0, undef $vgpr1
- ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.2, addrspace 5)
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc
- ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.2, addrspace 5)
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc
- ; CHECK-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.2, addrspace 5)
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc, implicit killed $vgpr1
; CHECK-NEXT: $vgpr0 = V_ADD_U32_e32 64, killed $vgpr0, implicit $exec
- ; CHECK-NEXT: $sgpr4 = V_READFIRSTLANE_B32 $vgpr0, implicit $exec
- ; CHECK-NEXT: renamable $vcc_lo = S_CSELECT_B32 killed $sgpr4, renamable $sgpr97, implicit $scc
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc, implicit-def $vgpr0
- ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.2, addrspace 5)
- ; CHECK-NEXT: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.2, addrspace 5)
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc
- ; CHECK-NEXT: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.2, addrspace 5)
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc
- ; CHECK-NEXT: $sgpr4 = SI_RESTORE_S32_FROM_VGPR killed $vgpr0, 0
- ; CHECK-NEXT: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.2, addrspace 5)
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc, implicit killed $vgpr0
+ ; CHECK-NEXT: $sgpr32 = V_READFIRSTLANE_B32 $vgpr0, implicit $exec
+ ; CHECK-NEXT: renamable $vcc_lo = S_CSELECT_B32 $sgpr32, renamable $sgpr97, implicit $scc
+ ; CHECK-NEXT: $sgpr32 = S_MUL_I32 $sgpr32, 64
+ ; CHECK-NEXT: $sgpr32 = S_ADDC_U32 $sgpr32, -4096, implicit-def $scc, implicit $scc
+ ; CHECK-NEXT: S_BITCMP1_B32 $sgpr32, 0, implicit-def $scc
+ ; CHECK-NEXT: $sgpr32 = S_BITSET0_B32 0, $sgpr32
; CHECK-NEXT: S_NOP 0, implicit killed $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, implicit killed $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31_sgpr32_sgpr33_sgpr34_sgpr35, implicit killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, implicit killed $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, implicit killed $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, implicit killed $sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, implicit killed $sgpr96_sgpr97_sgpr98_sgpr99
; CHECK-NEXT: S_CBRANCH_SCC1 %bb.2, implicit $scc
; CHECK-NEXT: S_BRANCH %bb.2
@@ -328,8 +295,9 @@ body: |
---
# As above, but with a non-zero offset and with SCC dead after the frame index
-# user: S_CSELECT_B32 is its last read. The emergency spill clobbers SCC before
-# that read here too.
+# user: S_CSELECT_B32 is its last read. FrameReg is still restored with the
+# S_ADDC_U32/S_BITCMP1_B32/S_BITSET0_B32 sequence that preserves SCC, even
+# though nothing after MI reads it.
name: scc_live_into_dead_after_scalar_frame_index_no_free_sgpr_nonzero_offset
tracksRegLiveness: true
stack:
@@ -357,27 +325,13 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_CMP_EQ_U32 renamable $sgpr96, 0, implicit-def $scc
; CHECK-NEXT: $vgpr0 = V_LSHRREV_B32_e64 6, $sgpr32, implicit $exec
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc, implicit-def $vgpr1
- ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.2, addrspace 5)
- ; CHECK-NEXT: $vgpr1 = SI_SPILL_S32_TO_VGPR killed $sgpr4, 0, undef $vgpr1
- ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.2, addrspace 5)
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc
- ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.2, addrspace 5)
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc
- ; CHECK-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.2, addrspace 5)
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc, implicit killed $vgpr1
; CHECK-NEXT: $vgpr0 = V_ADD_U32_e32 64, killed $vgpr0, implicit $exec
- ; CHECK-NEXT: $sgpr4 = V_READFIRSTLANE_B32 $vgpr0, implicit $exec
- ; CHECK-NEXT: renamable $vcc_lo = S_CSELECT_B32 killed $sgpr4, renamable $sgpr97, implicit $scc
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc, implicit-def $vgpr0
- ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" store (s32) into %stack.2, addrspace 5)
- ; CHECK-NEXT: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.2, addrspace 5)
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc
- ; CHECK-NEXT: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.2, addrspace 5)
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc
- ; CHECK-NEXT: $sgpr4 = SI_RESTORE_S32_FROM_VGPR killed $vgpr0, 0
- ; CHECK-NEXT: $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 128, 0, 0, implicit $exec :: ("amdgpu-thread-private" load (s32) from %stack.2, addrspace 5)
- ; CHECK-NEXT: $exec = S_NOT_B64 $exec, implicit-def dead $scc, implicit killed $vgpr0
+ ; CHECK-NEXT: $sgpr32 = V_READFIRSTLANE_B32 $vgpr0, implicit $exec
+ ; CHECK-NEXT: renamable $vcc_lo = S_CSELECT_B32 $sgpr32, renamable $sgpr97, implicit $scc
+ ; CHECK-NEXT: $sgpr32 = S_MUL_I32 $sgpr32, 64
+ ; CHECK-NEXT: $sgpr32 = S_ADDC_U32 $sgpr32, -4096, implicit-def $scc, implicit $scc
+ ; CHECK-NEXT: S_BITCMP1_B32 $sgpr32, 0, implicit-def $scc
+ ; CHECK-NEXT: $sgpr32 = S_BITSET0_B32 0, $sgpr32
; CHECK-NEXT: S_NOP 0, implicit killed $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, implicit killed $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31_sgpr32_sgpr33_sgpr34_sgpr35, implicit killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, implicit killed $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, implicit killed $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, implicit killed $sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, implicit killed $sgpr96_sgpr97_sgpr98_sgpr99
; CHECK-NEXT: S_BRANCH %bb.2
; CHECK-NEXT: {{ $}}
>From a4a68ab76117435eb9b3e446b4ecff8da947168b Mon Sep 17 00:00:00 2001
From: Domenic Nutile <domenic.nutile at gmail.com>
Date: Mon, 21 Sep 2026 13:31:31 -0400
Subject: [PATCH 2/3] [AMDGPU] Use the scavenger to test whether SCC is live
after MI
The register scavenger is stepped backwards to the liveness state
immediately after MI, so RS->isRegUsed(SCC) already answers "is SCC live
after MI" directly. Replace the hand-rolled test with that query.
---
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 6 +++---
...r-spill-to-vmem-scc-clobber-reserved-exec-copy.mir | 11 +++++------
2 files changed, 8 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index b6b1bccb0820d..87e18dba86d4b 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -3409,6 +3409,9 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
// In an entry function/kernel the offset is already swizzled.
bool IsSALU = isSGPRClass(TII->getRegClass(MI->getDesc(), FIOperandNum));
bool LiveSCC = isSCCLiveInto(*RS, *MI);
+ // The scavenger is positioned at the liveness state immediately after MI,
+ // so we need only check if SCC is used.
+ bool SCCLiveAfterMI = RS->isRegUsed(AMDGPU::SCC);
const TargetRegisterClass *RC = IsSALU && !LiveSCC
? &AMDGPU::SReg_32RegClass
: &AMDGPU::VGPR_32RegClass;
@@ -3659,9 +3662,6 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
if (Offset) {
int64_t ScaledOffset = -Offset * ST.getWavefrontSize();
- bool SCCLiveAfterMI = MI->definesRegister(AMDGPU::SCC, this)
- ? !MI->registerDefIsDead(AMDGPU::SCC, this)
- : LiveSCC;
if (!SCCLiveAfterMI) {
BuildMI(*MBB, InsPt, DL, TII->get(AMDGPU::S_ADD_I32), FrameReg)
.addReg(FrameReg)
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-to-vmem-scc-clobber-reserved-exec-copy.mir b/llvm/test/CodeGen/AMDGPU/sgpr-spill-to-vmem-scc-clobber-reserved-exec-copy.mir
index 36a532d6c63e0..c499f00a57361 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-to-vmem-scc-clobber-reserved-exec-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-to-vmem-scc-clobber-reserved-exec-copy.mir
@@ -295,9 +295,10 @@ body: |
---
# As above, but with a non-zero offset and with SCC dead after the frame index
-# user: S_CSELECT_B32 is its last read. FrameReg is still restored with the
-# S_ADDC_U32/S_BITCMP1_B32/S_BITSET0_B32 sequence that preserves SCC, even
-# though nothing after MI reads it.
+# user: S_CSELECT_B32 is its last read. Restoring FrameReg can therefore fold
+# the offset back with a plain S_ADD_I32 and clobber SCC freely, instead of the
+# S_ADDC_U32/S_BITCMP1_B32/S_BITSET0_B32 sequence needed when SCC has a later
+# use.
name: scc_live_into_dead_after_scalar_frame_index_no_free_sgpr_nonzero_offset
tracksRegLiveness: true
stack:
@@ -329,9 +330,7 @@ body: |
; CHECK-NEXT: $sgpr32 = V_READFIRSTLANE_B32 $vgpr0, implicit $exec
; CHECK-NEXT: renamable $vcc_lo = S_CSELECT_B32 $sgpr32, renamable $sgpr97, implicit $scc
; CHECK-NEXT: $sgpr32 = S_MUL_I32 $sgpr32, 64
- ; CHECK-NEXT: $sgpr32 = S_ADDC_U32 $sgpr32, -4096, implicit-def $scc, implicit $scc
- ; CHECK-NEXT: S_BITCMP1_B32 $sgpr32, 0, implicit-def $scc
- ; CHECK-NEXT: $sgpr32 = S_BITSET0_B32 0, $sgpr32
+ ; CHECK-NEXT: $sgpr32 = S_ADD_I32 $sgpr32, -4096, implicit-def $scc
; CHECK-NEXT: S_NOP 0, implicit killed $sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19, implicit killed $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31_sgpr32_sgpr33_sgpr34_sgpr35, implicit killed $sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51, implicit killed $sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67, implicit killed $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83, implicit killed $sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, implicit killed $sgpr96_sgpr97_sgpr98_sgpr99
; CHECK-NEXT: S_BRANCH %bb.2
; CHECK-NEXT: {{ $}}
>From 70e1d57e3a54f280dd4015d069ca258949f57ad0 Mon Sep 17 00:00:00 2001
From: Domenic Nutile <domenic.nutile at gmail.com>
Date: Mon, 21 Sep 2026 15:18:15 -0400
Subject: [PATCH 3/3] [AMDGPU] Update based on review feedback
Replace the lambda with the check inlined at both sites, and report a fatal
error when neither a free SGPR nor FrameReg is available, rather than
silently falling back to the spilling scavenge.
---
llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp | 43 +++++++++++++----------
1 file changed, 24 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 87e18dba86d4b..ec324d3667032 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -3433,23 +3433,6 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
bool RestoreFrameReg = false;
- // Scavenge the scalar temporary the V_READFIRSTLANE_B32 result lands in.
- // Spilling an SGPR here would flip EXEC with S_NOT, and that clobbers the
- // SCC this path exists to preserve, so fall back to FrameReg instead.
- auto ScavengeSGPRForReadfirstlane =
- [&](MachineBasicBlock::iterator To) -> Register {
- if (Register Reg = RS->scavengeRegisterBackwards(
- AMDGPU::SReg_32_XM0RegClass, To, false, 0,
- /*AllowSpill=*/false))
- return Reg;
- if (CanUseFrameRegAsSGPRScratch) {
- RestoreFrameReg = true;
- return FrameReg;
- }
- return RS->scavengeRegisterBackwards(AMDGPU::SReg_32_XM0RegClass, To,
- false, 0);
- };
-
Register ResultReg;
if (IsCopy) {
ResultReg = MI->getOperand(0).getReg();
@@ -3503,7 +3486,19 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
assert(ResultReg.isPhysical());
NewDest = ResultReg;
} else {
- NewDest = ScavengeSGPRForReadfirstlane(Shift);
+ // Spilling an SGPR here would flip EXEC with S_NOT, and that
+ // clobbers the SCC this path exists to preserve, so scale FrameReg
+ // in place instead.
+ NewDest = RS->scavengeRegisterBackwards(AMDGPU::SReg_32_XM0RegClass,
+ Shift, false, 0,
+ /*AllowSpill=*/false);
+ if (!NewDest) {
+ if (!CanUseFrameRegAsSGPRScratch)
+ report_fatal_error("Cannot scavenge register while SCC is live "
+ "in FI elimination!");
+ NewDest = FrameReg;
+ RestoreFrameReg = true;
+ }
}
BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), NewDest)
.addReg(TmpResultReg);
@@ -3623,7 +3618,17 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
if (IsCopy) {
NewDest = ResultReg;
} else {
- NewDest = ScavengeSGPRForReadfirstlane(*Add);
+ // As above, an SGPR spill here would clobber the live SCC.
+ NewDest = RS->scavengeRegisterBackwards(
+ AMDGPU::SReg_32_XM0RegClass, *Add, false, 0,
+ /*AllowSpill=*/false);
+ if (!NewDest) {
+ if (!CanUseFrameRegAsSGPRScratch)
+ report_fatal_error("Cannot scavenge register while SCC is "
+ "live in FI elimination!");
+ NewDest = FrameReg;
+ RestoreFrameReg = true;
+ }
}
BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32),
More information about the llvm-branch-commits
mailing list