[llvm] [AMDGPU] Handle register variant S_SETREG when fixing VGPR MSB clobbers (PR #218941)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 26 07:46:33 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Arseniy Obolenskiy (aobolensk)
<details>
<summary>Changes</summary>
Its value is unknown at compile time, so the clobbered VGPR MSBs must always be restored
---
Full diff: https://github.com/llvm/llvm-project/pull/218941.diff
5 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp (+33-16)
- (modified) llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp (+4-15)
- (modified) llvm/lib/Target/AMDGPU/SIInstrInfo.h (+13)
- (added) llvm/test/CodeGen/AMDGPU/vgpr-setreg-mode-swar.ll (+27)
- (modified) llvm/test/CodeGen/AMDGPU/vgpr-setreg-mode-swar.mir (+122)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
index 75c3dd3b1de09..ccacca66f5fcf 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerVGPREncoding.cpp
@@ -197,20 +197,24 @@ class AMDGPULowerVGPREncoding {
MachineBasicBlock::instr_iterator
handleCoissue(MachineBasicBlock::instr_iterator I);
- /// S_SET_VGPR_MSB immediately after S_SETREG_IMM32_B32 targeting MODE is
+ /// S_SET_VGPR_MSB immediately after an S_SETREG variant targeting MODE is
/// silently dropped on GFX1250. When set, the next S_SET_VGPR_MSB insertion
/// must be preceded by S_NOP to avoid the hazard.
bool needNopBeforeSetVGPRMSB(MachineBasicBlock::instr_iterator I);
- /// Handle S_SETREG_IMM32_B32 targeting MODE register. On certain hardware,
+ /// Handle an S_SETREG variant targeting MODE register. On certain hardware,
/// this instruction clobbers VGPR MSB bits[12:19], so we need to restore
/// the current mode. \returns true if the instruction was modified or a
/// new one was inserted.
bool handleSetregMode(MachineInstr &MI);
- /// Update bits[12:19] of the imm operand in S_SETREG_IMM32_B32 to contain
- /// the VGPR MSB mode value. \returns true if the immediate was changed.
+ /// Update bits[12:19] of the imm operand in an S_SETREG_IMM32_B32 variant to
+ /// contain the VGPR MSB mode value. \returns true if the immediate was
+ /// changed.
bool updateSetregModeImm(MachineInstr &MI, int64_t ModeValue);
+
+ /// Insert S_NOP + S_SET_VGPR_MSB restoring \p ModeValue after setreg \p MI.
+ void restoreModeAfterSetreg(MachineInstr &MI, int64_t ModeValue);
};
bool AMDGPULowerVGPREncoding::setMode(ModeTy NewMode,
@@ -433,9 +437,9 @@ AMDGPULowerVGPREncoding::handleCoissue(MachineBasicBlock::instr_iterator I) {
return I;
}
-/// Returns whether \p MI is a S_SETREG_IMM32_B32(MODE).
+/// Returns whether \p MI is an S_SETREG variant targeting MODE.
static bool isSetregMode(const MachineInstr &MI, const SIInstrInfo &TII) {
- if (MI.getOpcode() != AMDGPU::S_SETREG_IMM32_B32)
+ if (!SIInstrInfo::isSSetReg(MI.getOpcode()))
return false;
const MachineOperand *SIMM16Op =
@@ -460,7 +464,7 @@ bool AMDGPULowerVGPREncoding::needNopBeforeSetVGPRMSB(
}
// Look for a potential fallthrough predecessor block. When it ends with a
- // S_SETREG_IMM32_B32(MODE) we need to insert a S_NOP too. We assume that an
+ // setreg targeting MODE we need to insert a S_NOP too. We assume that an
// explicit jump to the current block from the block that would otherwise
// have naturally fallen through to it will remain in the final assembly.
CurrentMBB = CurrentMBB->getPrevNode();
@@ -482,7 +486,8 @@ static int64_t convertModeToSetregFormat(int64_t Mode) {
bool AMDGPULowerVGPREncoding::updateSetregModeImm(MachineInstr &MI,
int64_t ModeValue) {
- assert(MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32);
+ assert(MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
+ MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32_mode);
// Convert from S_SET_VGPR_MSB format to MODE register format
int64_t SetregMode = convertModeToSetregFormat(ModeValue);
@@ -499,8 +504,8 @@ bool AMDGPULowerVGPREncoding::updateSetregModeImm(MachineInstr &MI,
bool AMDGPULowerVGPREncoding::handleSetregMode(MachineInstr &MI) {
using namespace AMDGPU::Hwreg;
- assert(MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 &&
- "only S_SETREG_IMM32_B32 needs to be handled");
+ assert(SIInstrInfo::isSSetReg(MI.getOpcode()) &&
+ "expected an S_SETREG variant");
LLVM_DEBUG(dbgs() << " handleSetregMode: " << MI);
@@ -526,6 +531,14 @@ bool AMDGPULowerVGPREncoding::handleSetregMode(MachineInstr &MI) {
<< " VGPRMSBShift=" << VGPRMSBShift << '\n';
});
+ // Register variants write an SGPR to MODE, so the resulting VGPR MSBs are
+ // unknown at compile time.
+ if (MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
+ MI.getOpcode() == AMDGPU::S_SETREG_B32_mode) {
+ restoreModeAfterSetreg(MI, ModeValue);
+ return true;
+ }
+
// Case 1: Size <= 12 - the original instruction uses imm32[0:Size-1], so
// imm32[12:19] is unused, or Offset is zero and it is safe to set
// imm32[12:19] to the correct VGPR MSBs.
@@ -555,10 +568,15 @@ bool AMDGPULowerVGPREncoding::handleSetregMode(MachineInstr &MI) {
return false;
}
- // imm32[12:19] doesn't match VGPR MSBs - insert s_set_vgpr_msb after
- // the original instruction to restore the correct value. Insert S_NOP
- // to avoid the GFX1250 hazard where S_SET_VGPR_MSB immediately after
- // S_SETREG_IMM32_B32(MODE) is silently dropped.
+ // imm32[12:19] doesn't match VGPR MSBs - restore the correct value after MI.
+ restoreModeAfterSetreg(MI, ModeValue);
+ return true;
+}
+
+void AMDGPULowerVGPREncoding::restoreModeAfterSetreg(MachineInstr &MI,
+ int64_t ModeValue) {
+ // S_NOP avoids the GFX1250 hazard: S_SET_VGPR_MSB right after a MODE setreg
+ // is silently dropped.
MachineBasicBlock::iterator InsertPt = std::next(MI.getIterator());
BuildMI(*MBB, InsertPt, MI.getDebugLoc(), TII->get(AMDGPU::S_NOP)).addImm(0);
MostRecentModeSet = BuildMI(*MBB, InsertPt, MI.getDebugLoc(),
@@ -566,7 +584,6 @@ bool AMDGPULowerVGPREncoding::handleSetregMode(MachineInstr &MI) {
.addImm(ModeValue | (ModeValue << ModeWidth));
LLVM_DEBUG(dbgs() << " -> inserted S_SET_VGPR_MSB after setreg: "
<< *MostRecentModeSet);
- return true;
}
bool AMDGPULowerVGPREncoding::run(MachineFunction &MF) {
@@ -623,7 +640,7 @@ bool AMDGPULowerVGPREncoding::run(MachineFunction &MF) {
continue;
}
- if (MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 &&
+ if (SIInstrInfo::isSSetReg(MI.getOpcode()) &&
ST.hasSetregVGPRMSBFixup()) {
Changed |= handleSetregMode(MI);
continue;
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index 9d01a529eb012..3c0d9656dc4ab 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -459,17 +459,6 @@ static bool isSGetReg(unsigned Opcode) {
return Opcode == AMDGPU::S_GETREG_B32 || Opcode == AMDGPU::S_GETREG_B32_const;
}
-static bool isSSetReg(unsigned Opcode) {
- switch (Opcode) {
- case AMDGPU::S_SETREG_B32:
- case AMDGPU::S_SETREG_B32_mode:
- case AMDGPU::S_SETREG_IMM32_B32:
- case AMDGPU::S_SETREG_IMM32_B32_mode:
- return true;
- }
- return false;
-}
-
static bool isRWLane(unsigned Opcode) {
return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
}
@@ -615,7 +604,7 @@ GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
if (isSGetReg(MI->getOpcode()) && checkGetRegHazards(MI) > 0)
return HazardType;
- if (isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0)
+ if (SIInstrInfo::isSSetReg(MI->getOpcode()) && checkSetRegHazards(MI) > 0)
return HazardType;
if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0)
@@ -773,7 +762,7 @@ unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) const {
if (isSGetReg(MI->getOpcode()))
return std::max(WaitStates, checkGetRegHazards(MI));
- if (isSSetReg(MI->getOpcode()))
+ if (SIInstrInfo::isSSetReg(MI->getOpcode()))
return std::max(WaitStates, checkSetRegHazards(MI));
if (isRFE(MI->getOpcode()))
@@ -1094,7 +1083,7 @@ int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg,
int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
int Limit) const {
auto IsHazardFn = [IsHazard](const MachineInstr &MI) {
- return isSSetReg(MI.getOpcode()) && IsHazard(MI);
+ return SIInstrInfo::isSSetReg(MI.getOpcode()) && IsHazard(MI);
};
return getWaitStatesSince(IsHazardFn, Limit);
@@ -4300,7 +4289,7 @@ bool GCNHazardRecognizer::fixScratchBaseForwardingHazard(MachineInstr *MI) {
}
bool GCNHazardRecognizer::fixSetRegMode(MachineInstr *MI) {
- if (!isSSetReg(MI->getOpcode()) ||
+ if (!SIInstrInfo::isSSetReg(MI->getOpcode()) ||
MI->getOperand(1).getImm() != AMDGPU::Hwreg::ID_MODE)
return false;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index a62a753b41b08..b49524be99c98 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1252,6 +1252,19 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
}
}
+ /// \returns true if \p Opcode is any S_SETREG_B32/S_SETREG_IMM32_B32 variant.
+ static bool isSSetReg(unsigned Opcode) {
+ switch (Opcode) {
+ case AMDGPU::S_SETREG_B32:
+ case AMDGPU::S_SETREG_B32_mode:
+ case AMDGPU::S_SETREG_IMM32_B32:
+ case AMDGPU::S_SETREG_IMM32_B32_mode:
+ return true;
+ default:
+ return false;
+ }
+ }
+
bool isVGPRCopy(const MachineInstr &MI) const {
assert(isCopyInstr(MI));
Register Dest = MI.getOperand(0).getReg();
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-setreg-mode-swar.ll b/llvm/test/CodeGen/AMDGPU/vgpr-setreg-mode-swar.ll
new file mode 100644
index 0000000000000..ab53e3fa6f19c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-setreg-mode-swar.ll
@@ -0,0 +1,27 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 < %s | FileCheck %s
+
+; llvm.set.rounding lowers to s_setreg_b32(MODE), leaving garbage in
+; bits[19:12] of the SGPR operand, so the tracked mode must be re-established.
+
+declare void @llvm.set.rounding(i32)
+
+; CHECK-LABEL: {{^}}dynamic_rounding_mode_with_high_vgprs:
+; CHECK: s_setreg_b32 hwreg(HW_REG_WAVE_MODE, 0, 4), s0
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: s_set_vgpr_msb 0x4141
+define amdgpu_kernel void @dynamic_rounding_mode_with_high_vgprs(ptr addrspace(1) %p, ptr addrspace(1) %q, i32 %mode) #0 {
+ %a = load volatile <64 x float>, ptr addrspace(1) %p
+ %b = load volatile <64 x float>, ptr addrspace(1) %p
+ %c = load volatile <64 x float>, ptr addrspace(1) %p
+ %d = load volatile <64 x float>, ptr addrspace(1) %p
+ %e = load volatile <64 x float>, ptr addrspace(1) %p
+ store volatile <64 x float> %a, ptr addrspace(1) %q
+ call void @llvm.set.rounding(i32 %mode)
+ %sum = call <64 x float> @llvm.experimental.constrained.fadd.v64f32(<64 x float> %b, <64 x float> %c, metadata !"round.dynamic", metadata !"fpexcept.strict")
+ store volatile <64 x float> %sum, ptr addrspace(1) %q
+ store volatile <64 x float> %d, ptr addrspace(1) %q
+ store volatile <64 x float> %e, ptr addrspace(1) %q
+ ret void
+}
+
+attributes #0 = { strictfp "amdgpu-flat-work-group-size"="1,32" "amdgpu-waves-per-eu"="1,1" }
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-setreg-mode-swar.mir b/llvm/test/CodeGen/AMDGPU/vgpr-setreg-mode-swar.mir
index b9c3cdb3db45f..b0a0143325f07 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-setreg-mode-swar.mir
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-setreg-mode-swar.mir
@@ -517,3 +517,125 @@ body: |
S_SETREG_IMM32_B32 23228, 30785, implicit-def $mode, implicit $mode
S_ENDPGM 0
...
+
+---
+# S_SETREG_IMM32_B32_mode clobbers VGPR MSBs the same way as S_SETREG_IMM32_B32.
+
+# ASM-LABEL: {{^}}setreg_imm32_mode_variant:
+# ASM: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 0, 4), 0x5005
+
+# DIS-LABEL: <setreg_imm32_mode_variant>:
+# DIS: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 0, 4), 0x5005
+# DIS-NEXT: v_mov_b32_e32 v2 /*v258*/, v3 /*v259*/
+
+name: setreg_imm32_mode_variant
+tracksRegLiveness: true
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: setreg_imm32_mode_variant
+ ; CHECK: S_SET_VGPR_MSB 65, implicit-def $mode
+ ; CHECK-NEXT: $vgpr256 = V_MOV_B32_e32 $vgpr257, implicit $exec
+ ; CHECK-NEXT: S_SETREG_IMM32_B32_mode 20485, 6145, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: $vgpr258 = V_MOV_B32_e32 $vgpr259, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr256 = V_MOV_B32_e32 $vgpr257, implicit $exec
+ ; size=4, offset=0, hwreg=MODE: simm16 = 0x1801 = 6145
+ S_SETREG_IMM32_B32_mode 5, 6145, implicit-def $mode, implicit $mode
+ $vgpr258 = V_MOV_B32_e32 $vgpr259, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+# Register variants write an SGPR to MODE, so the VGPR MSBs are unknown at
+# compile time and must always be restored.
+
+# ASM-LABEL: {{^}}setreg_sgpr:
+# ASM: s_setreg_b32 hwreg(HW_REG_WAVE_MODE, 0, 4), s0
+# ASM-NEXT: s_nop 0
+# ASM-NEXT: s_set_vgpr_msb 0x4141
+
+# DIS-LABEL: <setreg_sgpr>:
+# DIS: s_setreg_b32 hwreg(HW_REG_WAVE_MODE, 0, 4), s0
+# DIS-NEXT: s_nop 0
+# DIS-NEXT: s_set_vgpr_msb 0x4141
+# DIS-NEXT: v_mov_b32_e32 v2 /*v258*/, v3 /*v259*/
+
+name: setreg_sgpr
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: setreg_sgpr
+ ; CHECK: S_SET_VGPR_MSB 65, implicit-def $mode
+ ; CHECK-NEXT: $vgpr256 = V_MOV_B32_e32 $vgpr257, implicit $exec
+ ; CHECK-NEXT: S_SETREG_B32 $sgpr0, 6145, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_NOP 0
+ ; CHECK-NEXT: S_SET_VGPR_MSB 16705, implicit-def $mode
+ ; CHECK-NEXT: $vgpr258 = V_MOV_B32_e32 $vgpr259, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr256 = V_MOV_B32_e32 $vgpr257, implicit $exec
+ ; size=4, offset=0, hwreg=MODE: simm16 = 0x1801 = 6145
+ S_SETREG_B32 $sgpr0, 6145, implicit-def $mode, implicit $mode
+ $vgpr258 = V_MOV_B32_e32 $vgpr259, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+# ASM-LABEL: {{^}}setreg_sgpr_mode_variant:
+# ASM: s_setreg_b32 hwreg(HW_REG_WAVE_MODE, 0, 4), s0
+# ASM-NEXT: s_nop 0
+# ASM-NEXT: s_set_vgpr_msb 0x4141
+
+# DIS-LABEL: <setreg_sgpr_mode_variant>:
+# DIS: s_setreg_b32 hwreg(HW_REG_WAVE_MODE, 0, 4), s0
+# DIS-NEXT: s_nop 0
+# DIS-NEXT: s_set_vgpr_msb 0x4141
+# DIS-NEXT: v_mov_b32_e32 v2 /*v258*/, v3 /*v259*/
+
+name: setreg_sgpr_mode_variant
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: setreg_sgpr_mode_variant
+ ; CHECK: S_SET_VGPR_MSB 65, implicit-def $mode
+ ; CHECK-NEXT: $vgpr256 = V_MOV_B32_e32 $vgpr257, implicit $exec
+ ; CHECK-NEXT: S_SETREG_B32_mode $sgpr0, 6145, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_NOP 0
+ ; CHECK-NEXT: S_SET_VGPR_MSB 16705, implicit-def $mode
+ ; CHECK-NEXT: $vgpr258 = V_MOV_B32_e32 $vgpr259, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr256 = V_MOV_B32_e32 $vgpr257, implicit $exec
+ ; size=4, offset=0, hwreg=MODE: simm16 = 0x1801 = 6145
+ S_SETREG_B32_mode $sgpr0, 6145, implicit-def $mode, implicit $mode
+ $vgpr258 = V_MOV_B32_e32 $vgpr259, implicit $exec
+ S_ENDPGM 0
+...
+
+---
+# A non-MODE register setreg must be left alone.
+
+# ASM-LABEL: {{^}}setreg_sgpr_non_mode:
+# ASM: s_setreg_b32 hwreg(HW_REG_WAVE_STATUS, 0, 4), s0
+# ASM-NOT: s_set_vgpr_msb
+# ASM: s_endpgm
+
+# DIS-LABEL: <setreg_sgpr_non_mode>:
+# DIS: s_setreg_b32 hwreg(HW_REG_WAVE_STATUS, 0, 4), s0
+# DIS-NEXT: s_endpgm
+
+name: setreg_sgpr_non_mode
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr0
+ ; CHECK-LABEL: name: setreg_sgpr_non_mode
+ ; CHECK: S_SET_VGPR_MSB 65, implicit-def $mode
+ ; CHECK-NEXT: $vgpr256 = V_MOV_B32_e32 $vgpr257, implicit $exec
+ ; CHECK-NEXT: S_SETREG_B32 $sgpr0, 6146, implicit-def $mode, implicit $mode
+ ; CHECK-NEXT: S_ENDPGM 0
+ $vgpr256 = V_MOV_B32_e32 $vgpr257, implicit $exec
+ ; size=4, offset=0, hwreg=STATUS: simm16 = 0x1802 = 6146
+ S_SETREG_B32 $sgpr0, 6146, implicit-def $mode, implicit $mode
+ S_ENDPGM 0
+...
``````````
</details>
https://github.com/llvm/llvm-project/pull/218941
More information about the llvm-commits
mailing list