[llvm] [AMDGPU] Restore the default rounding mode before calls and returns (PR #222247)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 8 23:12:03 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Arseniy Obolenskiy (aobolensk)
<details>
<summary>Changes</summary>
SIModeRegister set a non-default mode for `llvm.fptrunc.round` and left it live across the call boundary, where the caller assumes the default
---
Full diff: https://github.com/llvm/llvm-project/pull/222247.diff
6 Files Affected:
- (modified) llvm/docs/AMDGPUUsage.rst (+5-2)
- (modified) llvm/lib/Target/AMDGPU/SIModeRegister.cpp (+86-17)
- (modified) llvm/test/CodeGen/AMDGPU/fneg-combines.legal.f16.ll (+2)
- (modified) llvm/test/CodeGen/AMDGPU/fsub-as-fneg-src-modifier.ll (+4)
- (added) llvm/test/CodeGen/AMDGPU/mode-register-call-boundary.ll (+120)
- (modified) llvm/test/CodeGen/AMDGPU/v_mac_f16-fpdp-rounding-mode.ll (+1)
``````````diff
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index d09216f8fdc20..8a93998e2d048 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -20263,7 +20263,9 @@ On entry to a function:
#. GFX6-GFX8: M0 register set to the size of LDS in bytes. See
:ref:`amdgpu-amdhsa-kernel-prolog-m0`.
#. The EXEC register is set to the lanes active on entry to the function.
-#. MODE register: *TBD*
+#. MODE register: the floating point rounding mode fields hold the mode
+ requested by the program with ``llvm.set.rounding``, or the default, round
+ to nearest even. Other fields: *TBD*.
#. VGPR0-31 and SGPR4-29 are used to pass function input arguments as described
below.
#. SGPR30-31 return address (RA). The code address that the function must
@@ -20354,7 +20356,8 @@ On exit from a function:
their value.
#. The PC is set to the RA provided on entry.
-#. MODE register: *TBD*.
+#. MODE register: as on entry to the function, unless the function changed the
+ rounding mode on behalf of the program. Other fields: *TBD*.
#. All other registers are clobbered.
#. Any necessary ``s_waitcnt`` has been performed to ensure memory accessed by
function is available to the caller.
diff --git a/llvm/lib/Target/AMDGPU/SIModeRegister.cpp b/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
index dbe27a8030d4e..dd91122d4b171 100644
--- a/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
+++ b/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
@@ -15,8 +15,10 @@
//
#include "AMDGPU.h"
#include "GCNSubtarget.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
+#include <optional>
#include <queue>
#define DEBUG_TYPE "si-mode-register"
@@ -122,6 +124,8 @@ class SIModeRegister {
bool Changed = false;
+ bool EnforceCallBoundary = false;
+
bool run(MachineFunction &MF);
void processBlockPhase1(MachineBasicBlock &MBB, const SIInstrInfo *TII);
@@ -162,6 +166,71 @@ FunctionPass *llvm::createSIModeRegisterPass() {
return new SIModeRegisterLegacy();
}
+static bool isFPTruncRoundPseudo(const MachineInstr &MI) {
+ switch (MI.getOpcode()) {
+ case AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO:
+ case AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO_fake16_e32:
+ case AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO_t16_e64:
+ case AMDGPU::FPTRUNC_ROUND_F32_F64_PSEUDO:
+ case AMDGPU::FPTRUNC_ROUND_F16_F32_SALU_PSEUDO:
+ return true;
+ default:
+ return false;
+ }
+}
+
+// The opcodes for which getInstructionMode below returns a non-default Status.
+static bool mayNeedNonDefaultMode(const MachineInstr &MI) {
+ switch (MI.getOpcode()) {
+ case AMDGPU::V_INTERP_P1LL_F16:
+ case AMDGPU::V_INTERP_P1LV_F16:
+ case AMDGPU::V_INTERP_P2_F16:
+ return true;
+ default:
+ return isFPTruncRoundPseudo(MI);
+ }
+}
+
+// Returns the {offset, mask} of the mode field an explicit setreg writes.
+static std::optional<std::pair<unsigned, unsigned>>
+getModeSetregField(const MachineInstr &MI, const SIInstrInfo *TII) {
+ switch (MI.getOpcode()) {
+ case AMDGPU::S_SETREG_B32:
+ case AMDGPU::S_SETREG_B32_mode:
+ case AMDGPU::S_SETREG_IMM32_B32:
+ case AMDGPU::S_SETREG_IMM32_B32_mode:
+ break;
+ default:
+ return std::nullopt;
+ }
+ using namespace AMDGPU::Hwreg;
+ unsigned Dst = TII->getNamedOperand(MI, AMDGPU::OpName::simm16)->getImm();
+ auto [Id, Offset, Width] = HwregEncoding::decode(Dst);
+ if (Id != ID_MODE)
+ return std::nullopt;
+ return std::make_pair(Offset, maskTrailingOnes<unsigned>(Width) << Offset);
+}
+
+// A mode the program asked for outlives the function and is not undone.
+static bool writesRoundMode(const MachineInstr &MI, const SIInstrInfo *TII) {
+ if (MI.getOpcode() == AMDGPU::S_ROUND_MODE)
+ return true;
+ auto Field = getModeSetregField(MI, TII);
+ return Field && (Field->second & AMDGPU::Hwreg::FP_ROUND_MASK);
+}
+
+// Not the wave-ending opcodes or the epilog return. Tail calls: see isCall.
+static bool isReturnToCaller(const MachineInstr &MI) {
+ switch (MI.getOpcode()) {
+ case AMDGPU::SI_RETURN:
+ case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN:
+ case AMDGPU::S_SETPC_B64_return:
+ return true;
+ default:
+ return false;
+ }
+}
+
// Determine the Mode register setting required for this instruction.
// Instructions which don't use the Mode register return a null Status.
// Note this currently only deals with instructions that use the floating point
@@ -169,12 +238,11 @@ FunctionPass *llvm::createSIModeRegisterPass() {
Status SIModeRegister::getInstructionMode(MachineInstr &MI,
const SIInstrInfo *TII) {
unsigned Opcode = MI.getOpcode();
- if (TII->usesFPDPRounding(MI) ||
- Opcode == AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO ||
- Opcode == AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO_fake16_e32 ||
- Opcode == AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO_t16_e64 ||
- Opcode == AMDGPU::FPTRUNC_ROUND_F32_F64_PSEUDO ||
- Opcode == AMDGPU::FPTRUNC_ROUND_F16_F32_SALU_PSEUDO) {
+ // Caller and callee agree on the default mode (AMDGPUUsage.rst, "MODE
+ // register").
+ if (EnforceCallBoundary && (MI.isCall() || isReturnToCaller(MI)))
+ return DefaultStatus;
+ if (TII->usesFPDPRounding(MI) || isFPTruncRoundPseudo(MI)) {
switch (Opcode) {
case AMDGPU::V_INTERP_P1LL_F16:
case AMDGPU::V_INTERP_P1LV_F16:
@@ -272,20 +340,11 @@ void SIModeRegister::processBlockPhase1(MachineBasicBlock &MBB,
Status IPChange;
for (MachineInstr &MI : MBB) {
Status InstrMode = getInstructionMode(MI, TII);
- if (MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
- MI.getOpcode() == AMDGPU::S_SETREG_B32_mode ||
- MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
- MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32_mode) {
+ if (auto Field = getModeSetregField(MI, TII)) {
// We preserve any explicit mode register setreg instruction we encounter,
// as we assume it has been inserted by a higher authority (this is
// likely to be a very rare occurrence).
- unsigned Dst = TII->getNamedOperand(MI, AMDGPU::OpName::simm16)->getImm();
- using namespace AMDGPU::Hwreg;
- auto [Id, Offset, Width] = HwregEncoding::decode(Dst);
- if (Id != ID_MODE)
- continue;
-
- unsigned Mask = maskTrailingOnes<unsigned>(Width) << Offset;
+ auto [Offset, Mask] = *Field;
// If an InsertionPoint is set we will insert a setreg there.
if (InsertionPoint) {
@@ -460,6 +519,16 @@ bool SIModeRegister::run(MachineFunction &MF) {
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = ST.getInstrInfo();
+ // The mixed case is not handled: a restore would undo the program's request.
+ auto AnyInstr = [&MF](function_ref<bool(const MachineInstr &)> P) {
+ return any_of(
+ MF, [&P](const MachineBasicBlock &MBB) { return any_of(MBB, P); });
+ };
+ EnforceCallBoundary = AnyInstr(mayNeedNonDefaultMode) &&
+ !AnyInstr([TII](const MachineInstr &MI) {
+ return writesRoundMode(MI, TII);
+ });
+
// Processing is performed in a number of phases
// Phase 1 - determine the initial mode required by each block, and add setreg
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines.legal.f16.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines.legal.f16.ll
index e4f06251acf07..a2149569b588a 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-combines.legal.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-combines.legal.f16.ll
@@ -89,6 +89,7 @@ define { float, float } @v_fneg_interp_p1_f16(float %a, float %b) #0 {
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
; GCN-NEXT: v_interp_p1ll_f16 v0, v1, attr0.x
; GCN-NEXT: v_interp_p1ll_f16 v1, v1, attr0.y
+; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GCN-NEXT: s_setpc_b64 s[30:31]
%mul = fmul float %a, %b
%fneg = fneg float %mul
@@ -109,6 +110,7 @@ define { half, half } @v_fneg_interp_p2_f16(float %a, float %b) #0 {
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
; GCN-NEXT: v_interp_p2_f16 v0, v1, attr0.x, v2
; GCN-NEXT: v_interp_p2_f16 v1, v1, attr0.y, v2
+; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GCN-NEXT: s_setpc_b64 s[30:31]
%mul = fmul float %a, %b
%fneg = fneg float %mul
diff --git a/llvm/test/CodeGen/AMDGPU/fsub-as-fneg-src-modifier.ll b/llvm/test/CodeGen/AMDGPU/fsub-as-fneg-src-modifier.ll
index fd5ea6eab210e..9fb55ca7b14c4 100644
--- a/llvm/test/CodeGen/AMDGPU/fsub-as-fneg-src-modifier.ll
+++ b/llvm/test/CodeGen/AMDGPU/fsub-as-fneg-src-modifier.ll
@@ -1287,6 +1287,8 @@ define amdgpu_gfx float @fold_f16_fsub_into_fneg_modifier_interp_daz(float %v0,
; SDAG-NEXT: s_mov_b32 m0, s4
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
; SDAG-NEXT: v_interp_p1ll_f16 v0, -v0, attr2.y
+; SDAG-NEXT: s_nop 0
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-LABEL: fold_f16_fsub_into_fneg_modifier_interp_daz:
@@ -1296,6 +1298,8 @@ define amdgpu_gfx float @fold_f16_fsub_into_fneg_modifier_interp_daz(float %v0,
; GISEL-NEXT: s_mov_b32 m0, s4
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
; GISEL-NEXT: v_interp_p1ll_f16 v0, v0, attr2.y
+; GISEL-NEXT: s_nop 0
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GISEL-NEXT: s_setpc_b64 s[30:31]
%sub = fsub float -0.0, %v0
%p1_0 = call float @llvm.amdgcn.interp.p1.f16(float %sub, i32 1, i32 2, i1 0, i32 %m0)
diff --git a/llvm/test/CodeGen/AMDGPU/mode-register-call-boundary.ll b/llvm/test/CodeGen/AMDGPU/mode-register-call-boundary.ll
new file mode 100644
index 0000000000000..5b91b691a74b1
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/mode-register-call-boundary.ll
@@ -0,0 +1,120 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck %s
+
+; A mode SIModeRegister set for llvm.fptrunc.round is restored at a call or a
+; return. A mode the program asked for is not.
+
+declare void @sink(float)
+declare float @tail_target(float)
+
+define float @ret_restores_default(double %a) {
+; CHECK-LABEL: ret_restores_default:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
+; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %t = call float @llvm.fptrunc.round.f32.f64(double %a, metadata !"round.upward")
+ ret float %t
+}
+
+define float @tail_call_restores_default(double %a) {
+; CHECK-LABEL: tail_call_restores_default:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_getpc_b64 s[16:17]
+; CHECK-NEXT: s_add_u32 s16, s16, tail_target at gotpcrel32@lo+4
+; CHECK-NEXT: s_addc_u32 s17, s17, tail_target at gotpcrel32@hi+12
+; CHECK-NEXT: s_load_dwordx2 s[16:17], s[16:17], 0x0
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
+; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; CHECK-NEXT: s_setpc_b64 s[16:17]
+ %t = call float @llvm.fptrunc.round.f32.f64(double %a, metadata !"round.upward")
+ %u = tail call float @tail_target(float %t)
+ ret float %u
+}
+
+define void @call_restores_default(double %a) {
+; CHECK-LABEL: call_restores_default:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_mov_b32 s16, s33
+; CHECK-NEXT: s_mov_b32 s33, s32
+; CHECK-NEXT: s_or_saveexec_b64 s[18:19], -1
+; CHECK-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
+; CHECK-NEXT: s_mov_b64 exec, s[18:19]
+; CHECK-NEXT: v_writelane_b32 v40, s16, 2
+; CHECK-NEXT: v_writelane_b32 v40, s30, 0
+; CHECK-NEXT: s_addk_i32 s32, 0x400
+; CHECK-NEXT: v_writelane_b32 v40, s31, 1
+; CHECK-NEXT: s_getpc_b64 s[16:17]
+; CHECK-NEXT: s_add_u32 s16, s16, sink at gotpcrel32@lo+4
+; CHECK-NEXT: s_addc_u32 s17, s17, sink at gotpcrel32@hi+12
+; CHECK-NEXT: s_load_dwordx2 s[16:17], s[16:17], 0x0
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
+; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; CHECK-NEXT: v_readlane_b32 s30, v40, 0
+; CHECK-NEXT: v_readlane_b32 s31, v40, 1
+; CHECK-NEXT: s_mov_b32 s32, s33
+; CHECK-NEXT: v_readlane_b32 s4, v40, 2
+; CHECK-NEXT: s_or_saveexec_b64 s[6:7], -1
+; CHECK-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
+; CHECK-NEXT: s_mov_b64 exec, s[6:7]
+; CHECK-NEXT: s_mov_b32 s33, s4
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %t = call float @llvm.fptrunc.round.f32.f64(double %a, metadata !"round.upward")
+ call void @sink(float %t)
+ ret void
+}
+
+; s_endpgm ends the wave, so there is no caller to restore for.
+define amdgpu_kernel void @kernel_no_restore(ptr addrspace(1) %out, double %a) {
+; CHECK-LABEL: kernel_no_restore:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; CHECK-NEXT: v_mov_b32_e32 v2, 0
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: v_mov_b32_e32 v0, s2
+; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
+; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; CHECK-NEXT: global_store_dword v2, v0, s[0:1]
+; CHECK-NEXT: s_endpgm
+ %t = call float @llvm.fptrunc.round.f32.f64(double %a, metadata !"round.upward")
+ store float %t, ptr addrspace(1) %out
+ ret void
+}
+
+; A return to the shader epilog is not a call boundary either.
+define amdgpu_gs float @gs_no_restore(double %a) {
+; CHECK-LABEL: gs_no_restore:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
+; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; CHECK-NEXT: ; return to shader part epilog
+ %t = call float @llvm.fptrunc.round.f32.f64(double %a, metadata !"round.upward")
+ ret float %t
+}
+
+; An explicit setreg changes the mode for the caller, so no restore.
+define float @explicit_setreg_persists_with_fptrunc_round(double %a) {
+; CHECK-LABEL: explicit_setreg_persists_with_fptrunc_round:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 1
+; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 4), 15
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %t = call float @llvm.fptrunc.round.f32.f64(double %a, metadata !"round.upward")
+ call void @llvm.set.rounding(i32 0)
+ ret float %t
+}
diff --git a/llvm/test/CodeGen/AMDGPU/v_mac_f16-fpdp-rounding-mode.ll b/llvm/test/CodeGen/AMDGPU/v_mac_f16-fpdp-rounding-mode.ll
index b8813da69134f..7aba2d486750f 100644
--- a/llvm/test/CodeGen/AMDGPU/v_mac_f16-fpdp-rounding-mode.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_mac_f16-fpdp-rounding-mode.ll
@@ -16,6 +16,7 @@ define <2 x half> @v_mac_f16_fpdp_rounding(<2 x half> %a, <2 x half> %c, <2 x ha
; CHECK-NEXT: v_or_b32_e32 v0, v1, v6
; CHECK-NEXT: flat_store_short v[4:5], v3
; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; CHECK-NEXT: s_setpc_b64 s[30:31]
%round.x = call half @llvm.fptrunc.round.f16.f32(float %x, metadata !"round.upward")
%mul = fmul <2 x half> %a, %b
``````````
</details>
https://github.com/llvm/llvm-project/pull/222247
More information about the llvm-commits
mailing list