[llvm] [AMDGPU] Restore the default rounding mode before calls and returns (PR #222247)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 10 07:56:12 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/222247
>From a493d893bab158d9ab85dd3aa12057887268e592 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Wed, 9 Sep 2026 08:10:20 +0200
Subject: [PATCH 1/3] [AMDGPU] Restore the default rounding mode before calls
and returns
SIModeRegister set a non-default mode for `llvm.fptrunc.round` and left it live across the call boundary, where the caller assumes the default
---
llvm/docs/AMDGPUUsage.rst | 7 +-
llvm/lib/Target/AMDGPU/SIModeRegister.cpp | 103 ++++++++++++---
.../CodeGen/AMDGPU/fneg-combines.legal.f16.ll | 2 +
.../AMDGPU/fsub-as-fneg-src-modifier.ll | 4 +
.../AMDGPU/mode-register-call-boundary.ll | 120 ++++++++++++++++++
.../AMDGPU/v_mac_f16-fpdp-rounding-mode.ll | 1 +
6 files changed, 218 insertions(+), 19 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/mode-register-call-boundary.ll
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index d09216f8fdc20..8a93998e2d048 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -20263,7 +20263,9 @@ On entry to a function:
#. GFX6-GFX8: M0 register set to the size of LDS in bytes. See
:ref:`amdgpu-amdhsa-kernel-prolog-m0`.
#. The EXEC register is set to the lanes active on entry to the function.
-#. MODE register: *TBD*
+#. MODE register: the floating point rounding mode fields hold the mode
+ requested by the program with ``llvm.set.rounding``, or the default, round
+ to nearest even. Other fields: *TBD*.
#. VGPR0-31 and SGPR4-29 are used to pass function input arguments as described
below.
#. SGPR30-31 return address (RA). The code address that the function must
@@ -20354,7 +20356,8 @@ On exit from a function:
their value.
#. The PC is set to the RA provided on entry.
-#. MODE register: *TBD*.
+#. MODE register: as on entry to the function, unless the function changed the
+ rounding mode on behalf of the program. Other fields: *TBD*.
#. All other registers are clobbered.
#. Any necessary ``s_waitcnt`` has been performed to ensure memory accessed by
function is available to the caller.
diff --git a/llvm/lib/Target/AMDGPU/SIModeRegister.cpp b/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
index dbe27a8030d4e..dd91122d4b171 100644
--- a/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
+++ b/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
@@ -15,8 +15,10 @@
//
#include "AMDGPU.h"
#include "GCNSubtarget.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
+#include <optional>
#include <queue>
#define DEBUG_TYPE "si-mode-register"
@@ -122,6 +124,8 @@ class SIModeRegister {
bool Changed = false;
+ bool EnforceCallBoundary = false;
+
bool run(MachineFunction &MF);
void processBlockPhase1(MachineBasicBlock &MBB, const SIInstrInfo *TII);
@@ -162,6 +166,71 @@ FunctionPass *llvm::createSIModeRegisterPass() {
return new SIModeRegisterLegacy();
}
+static bool isFPTruncRoundPseudo(const MachineInstr &MI) {
+ switch (MI.getOpcode()) {
+ case AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO:
+ case AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO_fake16_e32:
+ case AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO_t16_e64:
+ case AMDGPU::FPTRUNC_ROUND_F32_F64_PSEUDO:
+ case AMDGPU::FPTRUNC_ROUND_F16_F32_SALU_PSEUDO:
+ return true;
+ default:
+ return false;
+ }
+}
+
+// The opcodes for which getInstructionMode below returns a non-default Status.
+static bool mayNeedNonDefaultMode(const MachineInstr &MI) {
+ switch (MI.getOpcode()) {
+ case AMDGPU::V_INTERP_P1LL_F16:
+ case AMDGPU::V_INTERP_P1LV_F16:
+ case AMDGPU::V_INTERP_P2_F16:
+ return true;
+ default:
+ return isFPTruncRoundPseudo(MI);
+ }
+}
+
+// Returns the {offset, mask} of the mode field an explicit setreg writes.
+static std::optional<std::pair<unsigned, unsigned>>
+getModeSetregField(const MachineInstr &MI, const SIInstrInfo *TII) {
+ switch (MI.getOpcode()) {
+ case AMDGPU::S_SETREG_B32:
+ case AMDGPU::S_SETREG_B32_mode:
+ case AMDGPU::S_SETREG_IMM32_B32:
+ case AMDGPU::S_SETREG_IMM32_B32_mode:
+ break;
+ default:
+ return std::nullopt;
+ }
+ using namespace AMDGPU::Hwreg;
+ unsigned Dst = TII->getNamedOperand(MI, AMDGPU::OpName::simm16)->getImm();
+ auto [Id, Offset, Width] = HwregEncoding::decode(Dst);
+ if (Id != ID_MODE)
+ return std::nullopt;
+ return std::make_pair(Offset, maskTrailingOnes<unsigned>(Width) << Offset);
+}
+
+// A mode the program asked for outlives the function and is not undone.
+static bool writesRoundMode(const MachineInstr &MI, const SIInstrInfo *TII) {
+ if (MI.getOpcode() == AMDGPU::S_ROUND_MODE)
+ return true;
+ auto Field = getModeSetregField(MI, TII);
+ return Field && (Field->second & AMDGPU::Hwreg::FP_ROUND_MASK);
+}
+
+// Not the wave-ending opcodes or the epilog return. Tail calls: see isCall.
+static bool isReturnToCaller(const MachineInstr &MI) {
+ switch (MI.getOpcode()) {
+ case AMDGPU::SI_RETURN:
+ case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN:
+ case AMDGPU::S_SETPC_B64_return:
+ return true;
+ default:
+ return false;
+ }
+}
+
// Determine the Mode register setting required for this instruction.
// Instructions which don't use the Mode register return a null Status.
// Note this currently only deals with instructions that use the floating point
@@ -169,12 +238,11 @@ FunctionPass *llvm::createSIModeRegisterPass() {
Status SIModeRegister::getInstructionMode(MachineInstr &MI,
const SIInstrInfo *TII) {
unsigned Opcode = MI.getOpcode();
- if (TII->usesFPDPRounding(MI) ||
- Opcode == AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO ||
- Opcode == AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO_fake16_e32 ||
- Opcode == AMDGPU::FPTRUNC_ROUND_F16_F32_PSEUDO_t16_e64 ||
- Opcode == AMDGPU::FPTRUNC_ROUND_F32_F64_PSEUDO ||
- Opcode == AMDGPU::FPTRUNC_ROUND_F16_F32_SALU_PSEUDO) {
+ // Caller and callee agree on the default mode (AMDGPUUsage.rst, "MODE
+ // register").
+ if (EnforceCallBoundary && (MI.isCall() || isReturnToCaller(MI)))
+ return DefaultStatus;
+ if (TII->usesFPDPRounding(MI) || isFPTruncRoundPseudo(MI)) {
switch (Opcode) {
case AMDGPU::V_INTERP_P1LL_F16:
case AMDGPU::V_INTERP_P1LV_F16:
@@ -272,20 +340,11 @@ void SIModeRegister::processBlockPhase1(MachineBasicBlock &MBB,
Status IPChange;
for (MachineInstr &MI : MBB) {
Status InstrMode = getInstructionMode(MI, TII);
- if (MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
- MI.getOpcode() == AMDGPU::S_SETREG_B32_mode ||
- MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
- MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32_mode) {
+ if (auto Field = getModeSetregField(MI, TII)) {
// We preserve any explicit mode register setreg instruction we encounter,
// as we assume it has been inserted by a higher authority (this is
// likely to be a very rare occurrence).
- unsigned Dst = TII->getNamedOperand(MI, AMDGPU::OpName::simm16)->getImm();
- using namespace AMDGPU::Hwreg;
- auto [Id, Offset, Width] = HwregEncoding::decode(Dst);
- if (Id != ID_MODE)
- continue;
-
- unsigned Mask = maskTrailingOnes<unsigned>(Width) << Offset;
+ auto [Offset, Mask] = *Field;
// If an InsertionPoint is set we will insert a setreg there.
if (InsertionPoint) {
@@ -460,6 +519,16 @@ bool SIModeRegister::run(MachineFunction &MF) {
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = ST.getInstrInfo();
+ // The mixed case is not handled: a restore would undo the program's request.
+ auto AnyInstr = [&MF](function_ref<bool(const MachineInstr &)> P) {
+ return any_of(
+ MF, [&P](const MachineBasicBlock &MBB) { return any_of(MBB, P); });
+ };
+ EnforceCallBoundary = AnyInstr(mayNeedNonDefaultMode) &&
+ !AnyInstr([TII](const MachineInstr &MI) {
+ return writesRoundMode(MI, TII);
+ });
+
// Processing is performed in a number of phases
// Phase 1 - determine the initial mode required by each block, and add setreg
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines.legal.f16.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines.legal.f16.ll
index e4f06251acf07..a2149569b588a 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-combines.legal.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-combines.legal.f16.ll
@@ -89,6 +89,7 @@ define { float, float } @v_fneg_interp_p1_f16(float %a, float %b) #0 {
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
; GCN-NEXT: v_interp_p1ll_f16 v0, v1, attr0.x
; GCN-NEXT: v_interp_p1ll_f16 v1, v1, attr0.y
+; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GCN-NEXT: s_setpc_b64 s[30:31]
%mul = fmul float %a, %b
%fneg = fneg float %mul
@@ -109,6 +110,7 @@ define { half, half } @v_fneg_interp_p2_f16(float %a, float %b) #0 {
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
; GCN-NEXT: v_interp_p2_f16 v0, v1, attr0.x, v2
; GCN-NEXT: v_interp_p2_f16 v1, v1, attr0.y, v2
+; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GCN-NEXT: s_setpc_b64 s[30:31]
%mul = fmul float %a, %b
%fneg = fneg float %mul
diff --git a/llvm/test/CodeGen/AMDGPU/fsub-as-fneg-src-modifier.ll b/llvm/test/CodeGen/AMDGPU/fsub-as-fneg-src-modifier.ll
index fd5ea6eab210e..9fb55ca7b14c4 100644
--- a/llvm/test/CodeGen/AMDGPU/fsub-as-fneg-src-modifier.ll
+++ b/llvm/test/CodeGen/AMDGPU/fsub-as-fneg-src-modifier.ll
@@ -1287,6 +1287,8 @@ define amdgpu_gfx float @fold_f16_fsub_into_fneg_modifier_interp_daz(float %v0,
; SDAG-NEXT: s_mov_b32 m0, s4
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
; SDAG-NEXT: v_interp_p1ll_f16 v0, -v0, attr2.y
+; SDAG-NEXT: s_nop 0
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-LABEL: fold_f16_fsub_into_fneg_modifier_interp_daz:
@@ -1296,6 +1298,8 @@ define amdgpu_gfx float @fold_f16_fsub_into_fneg_modifier_interp_daz(float %v0,
; GISEL-NEXT: s_mov_b32 m0, s4
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
; GISEL-NEXT: v_interp_p1ll_f16 v0, v0, attr2.y
+; GISEL-NEXT: s_nop 0
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GISEL-NEXT: s_setpc_b64 s[30:31]
%sub = fsub float -0.0, %v0
%p1_0 = call float @llvm.amdgcn.interp.p1.f16(float %sub, i32 1, i32 2, i1 0, i32 %m0)
diff --git a/llvm/test/CodeGen/AMDGPU/mode-register-call-boundary.ll b/llvm/test/CodeGen/AMDGPU/mode-register-call-boundary.ll
new file mode 100644
index 0000000000000..5b91b691a74b1
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/mode-register-call-boundary.ll
@@ -0,0 +1,120 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck %s
+
+; A mode SIModeRegister set for llvm.fptrunc.round is restored at a call or a
+; return. A mode the program asked for is not.
+
+declare void @sink(float)
+declare float @tail_target(float)
+
+define float @ret_restores_default(double %a) {
+; CHECK-LABEL: ret_restores_default:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
+; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %t = call float @llvm.fptrunc.round.f32.f64(double %a, metadata !"round.upward")
+ ret float %t
+}
+
+define float @tail_call_restores_default(double %a) {
+; CHECK-LABEL: tail_call_restores_default:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_getpc_b64 s[16:17]
+; CHECK-NEXT: s_add_u32 s16, s16, tail_target at gotpcrel32@lo+4
+; CHECK-NEXT: s_addc_u32 s17, s17, tail_target at gotpcrel32@hi+12
+; CHECK-NEXT: s_load_dwordx2 s[16:17], s[16:17], 0x0
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
+; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; CHECK-NEXT: s_setpc_b64 s[16:17]
+ %t = call float @llvm.fptrunc.round.f32.f64(double %a, metadata !"round.upward")
+ %u = tail call float @tail_target(float %t)
+ ret float %u
+}
+
+define void @call_restores_default(double %a) {
+; CHECK-LABEL: call_restores_default:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_mov_b32 s16, s33
+; CHECK-NEXT: s_mov_b32 s33, s32
+; CHECK-NEXT: s_or_saveexec_b64 s[18:19], -1
+; CHECK-NEXT: buffer_store_dword v40, off, s[0:3], s33 ; 4-byte Folded Spill
+; CHECK-NEXT: s_mov_b64 exec, s[18:19]
+; CHECK-NEXT: v_writelane_b32 v40, s16, 2
+; CHECK-NEXT: v_writelane_b32 v40, s30, 0
+; CHECK-NEXT: s_addk_i32 s32, 0x400
+; CHECK-NEXT: v_writelane_b32 v40, s31, 1
+; CHECK-NEXT: s_getpc_b64 s[16:17]
+; CHECK-NEXT: s_add_u32 s16, s16, sink at gotpcrel32@lo+4
+; CHECK-NEXT: s_addc_u32 s17, s17, sink at gotpcrel32@hi+12
+; CHECK-NEXT: s_load_dwordx2 s[16:17], s[16:17], 0x0
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
+; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; CHECK-NEXT: v_readlane_b32 s30, v40, 0
+; CHECK-NEXT: v_readlane_b32 s31, v40, 1
+; CHECK-NEXT: s_mov_b32 s32, s33
+; CHECK-NEXT: v_readlane_b32 s4, v40, 2
+; CHECK-NEXT: s_or_saveexec_b64 s[6:7], -1
+; CHECK-NEXT: buffer_load_dword v40, off, s[0:3], s33 ; 4-byte Folded Reload
+; CHECK-NEXT: s_mov_b64 exec, s[6:7]
+; CHECK-NEXT: s_mov_b32 s33, s4
+; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %t = call float @llvm.fptrunc.round.f32.f64(double %a, metadata !"round.upward")
+ call void @sink(float %t)
+ ret void
+}
+
+; s_endpgm ends the wave, so there is no caller to restore for.
+define amdgpu_kernel void @kernel_no_restore(ptr addrspace(1) %out, double %a) {
+; CHECK-LABEL: kernel_no_restore:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; CHECK-NEXT: v_mov_b32_e32 v2, 0
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: v_mov_b32_e32 v0, s2
+; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
+; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; CHECK-NEXT: global_store_dword v2, v0, s[0:1]
+; CHECK-NEXT: s_endpgm
+ %t = call float @llvm.fptrunc.round.f32.f64(double %a, metadata !"round.upward")
+ store float %t, ptr addrspace(1) %out
+ ret void
+}
+
+; A return to the shader epilog is not a call boundary either.
+define amdgpu_gs float @gs_no_restore(double %a) {
+; CHECK-LABEL: gs_no_restore:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
+; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; CHECK-NEXT: ; return to shader part epilog
+ %t = call float @llvm.fptrunc.round.f32.f64(double %a, metadata !"round.upward")
+ ret float %t
+}
+
+; An explicit setreg changes the mode for the caller, so no restore.
+define float @explicit_setreg_persists_with_fptrunc_round(double %a) {
+; CHECK-LABEL: explicit_setreg_persists_with_fptrunc_round:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 1
+; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 0, 4), 15
+; CHECK-NEXT: s_setpc_b64 s[30:31]
+ %t = call float @llvm.fptrunc.round.f32.f64(double %a, metadata !"round.upward")
+ call void @llvm.set.rounding(i32 0)
+ ret float %t
+}
diff --git a/llvm/test/CodeGen/AMDGPU/v_mac_f16-fpdp-rounding-mode.ll b/llvm/test/CodeGen/AMDGPU/v_mac_f16-fpdp-rounding-mode.ll
index b8813da69134f..7aba2d486750f 100644
--- a/llvm/test/CodeGen/AMDGPU/v_mac_f16-fpdp-rounding-mode.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_mac_f16-fpdp-rounding-mode.ll
@@ -16,6 +16,7 @@ define <2 x half> @v_mac_f16_fpdp_rounding(<2 x half> %a, <2 x half> %c, <2 x ha
; CHECK-NEXT: v_or_b32_e32 v0, v1, v6
; CHECK-NEXT: flat_store_short v[4:5], v3
; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; CHECK-NEXT: s_setpc_b64 s[30:31]
%round.x = call half @llvm.fptrunc.round.f16.f32(float %x, metadata !"round.upward")
%mul = fmul <2 x half> %a, %b
>From 51c336da36ba85032023392e3049115c2e782928 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Thu, 10 Sep 2026 16:31:07 +0200
Subject: [PATCH 2/3] refactor phase 1
---
llvm/lib/Target/AMDGPU/SIModeRegister.cpp | 69 ++++++++------
.../GlobalISel/llvm.amdgcn.interp.p1.f16.ll | 24 +++++
.../test/CodeGen/AMDGPU/llvm.fptrunc.round.ll | 94 +++++++++++++++++++
.../AMDGPU/mode-register-call-boundary.ll | 24 +++--
4 files changed, 173 insertions(+), 38 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIModeRegister.cpp b/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
index dd91122d4b171..ac17b8d607505 100644
--- a/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
+++ b/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
@@ -15,7 +15,6 @@
//
#include "AMDGPU.h"
#include "GCNSubtarget.h"
-#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include <optional>
@@ -99,6 +98,10 @@ class BlockData {
// which is used in Phase 3 if we need to insert a mode change.
MachineInstr *FirstInsertionPoint = nullptr;
+ // Call and return sites, each paired with the block local mode changes that
+ // precede it. Phase 3 merges the pair with Pred to get the mode at the site.
+ SmallVector<std::pair<MachineInstr *, Status>, 2> BoundarySites;
+
// A flag to indicate whether an Exit value has been set (we can't tell by
// examining the Exit value itself as all values may be valid results).
bool ExitSet = false;
@@ -124,7 +127,14 @@ class SIModeRegister {
bool Changed = false;
- bool EnforceCallBoundary = false;
+ // Set during Phase 1. A restore at a boundary would undo an explicit request,
+ // so the mixed case is not handled.
+ bool AnyNonDefaultMode = false;
+ bool AnyWritesRoundMode = false;
+
+ bool enforceCallBoundary() const {
+ return AnyNonDefaultMode && !AnyWritesRoundMode;
+ }
bool run(MachineFunction &MF);
@@ -136,7 +146,7 @@ class SIModeRegister {
Status getInstructionMode(MachineInstr &MI, const SIInstrInfo *TII);
- void insertSetreg(MachineBasicBlock &MBB, MachineInstr *I,
+ void insertSetreg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
const SIInstrInfo *TII, Status InstrMode);
};
@@ -211,18 +221,11 @@ getModeSetregField(const MachineInstr &MI, const SIInstrInfo *TII) {
return std::make_pair(Offset, maskTrailingOnes<unsigned>(Width) << Offset);
}
-// A mode the program asked for outlives the function and is not undone.
-static bool writesRoundMode(const MachineInstr &MI, const SIInstrInfo *TII) {
- if (MI.getOpcode() == AMDGPU::S_ROUND_MODE)
- return true;
- auto Field = getModeSetregField(MI, TII);
- return Field && (Field->second & AMDGPU::Hwreg::FP_ROUND_MASK);
-}
-
-// Not the wave-ending opcodes or the epilog return. Tail calls: see isCall.
+// Not the wave-ending opcodes. Tail calls: see isCall.
static bool isReturnToCaller(const MachineInstr &MI) {
switch (MI.getOpcode()) {
case AMDGPU::SI_RETURN:
+ case AMDGPU::SI_RETURN_TO_EPILOG:
case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN:
case AMDGPU::S_SETPC_B64_return:
return true;
@@ -238,10 +241,6 @@ static bool isReturnToCaller(const MachineInstr &MI) {
Status SIModeRegister::getInstructionMode(MachineInstr &MI,
const SIInstrInfo *TII) {
unsigned Opcode = MI.getOpcode();
- // Caller and callee agree on the default mode (AMDGPUUsage.rst, "MODE
- // register").
- if (EnforceCallBoundary && (MI.isCall() || isReturnToCaller(MI)))
- return DefaultStatus;
if (TII->usesFPDPRounding(MI) || isFPTruncRoundPseudo(MI)) {
switch (Opcode) {
case AMDGPU::V_INTERP_P1LL_F16:
@@ -292,14 +291,15 @@ Status SIModeRegister::getInstructionMode(MachineInstr &MI,
// the value of disjoint parts of the Mode register when we don't know the
// value of the intervening bits. In that case we need to use more than one
// setreg instruction.
-void SIModeRegister::insertSetreg(MachineBasicBlock &MBB, MachineInstr *MI,
+void SIModeRegister::insertSetreg(MachineBasicBlock &MBB,
+ MachineBasicBlock::iterator I,
const SIInstrInfo *TII, Status InstrMode) {
while (InstrMode.Mask) {
unsigned Offset = llvm::countr_zero<unsigned>(InstrMode.Mask);
unsigned Width = llvm::countr_one<unsigned>(InstrMode.Mask >> Offset);
unsigned Value = (InstrMode.Mode >> Offset) & ((1 << Width) - 1);
using namespace AMDGPU::Hwreg;
- BuildMI(MBB, MI, nullptr, TII->get(AMDGPU::S_SETREG_IMM32_B32))
+ BuildMI(MBB, I, nullptr, TII->get(AMDGPU::S_SETREG_IMM32_B32))
.addImm(Value)
.addImm(HwregEncoding::encode(ID_MODE, Offset, Width));
++NumSetregInserted;
@@ -339,12 +339,16 @@ void SIModeRegister::processBlockPhase1(MachineBasicBlock &MBB,
bool RequirePending = true;
Status IPChange;
for (MachineInstr &MI : MBB) {
+ // getInstructionMode rewrites the pseudos, so classify MI before the call.
+ AnyNonDefaultMode |= mayNeedNonDefaultMode(MI);
+ AnyWritesRoundMode |= MI.getOpcode() == AMDGPU::S_ROUND_MODE;
Status InstrMode = getInstructionMode(MI, TII);
if (auto Field = getModeSetregField(MI, TII)) {
// We preserve any explicit mode register setreg instruction we encounter,
// as we assume it has been inserted by a higher authority (this is
// likely to be a very rare occurrence).
auto [Offset, Mask] = *Field;
+ AnyWritesRoundMode |= (Mask & FP_ROUND_MODE_DP(0x3)) != 0;
// If an InsertionPoint is set we will insert a setreg there.
if (InsertionPoint) {
@@ -366,6 +370,11 @@ void SIModeRegister::processBlockPhase1(MachineBasicBlock &MBB,
} else {
NewInfo->Change = NewInfo->Change.mergeUnknown(Mask);
}
+ } else if (MI.isCall() || isReturnToCaller(MI)) {
+ // Whether a restore is needed is a whole function property, so only
+ // record the site here. Change is unaffected: Phase 3 pairs any restore
+ // it inserts with a re-set after the call.
+ NewInfo->BoundarySites.emplace_back(&MI, NewInfo->Change);
} else if (!NewInfo->Change.isCompatible(InstrMode)) {
// This instruction uses the Mode register and its requirements aren't
// compatible with the current mode.
@@ -491,6 +500,20 @@ void SIModeRegister::processBlockPhase3(MachineBasicBlock &MBB,
else
insertSetreg(MBB, &MBB.instr_front(), TII, Delta);
}
+
+ // Restore the default at the sites recorded in Phase 1.
+ if (!enforceCallBoundary())
+ return;
+ for (auto &[MI, ChangeAtSite] : BlockInfo[ThisBlock]->BoundarySites) {
+ Status AtSite = BlockInfo[ThisBlock]->Pred.merge(ChangeAtSite);
+ if (AtSite.isCompatible(DefaultStatus))
+ continue;
+ insertSetreg(MBB, MI, TII, AtSite.delta(DefaultStatus));
+ // Phase 1 modelled the site as mode preserving, so put the mode back.
+ if (!MI->isTerminator())
+ insertSetreg(MBB, std::next(MI->getIterator()), TII,
+ DefaultStatus.delta(AtSite));
+ }
}
bool SIModeRegisterLegacy::runOnMachineFunction(MachineFunction &MF) {
@@ -519,16 +542,6 @@ bool SIModeRegister::run(MachineFunction &MF) {
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
const SIInstrInfo *TII = ST.getInstrInfo();
- // The mixed case is not handled: a restore would undo the program's request.
- auto AnyInstr = [&MF](function_ref<bool(const MachineInstr &)> P) {
- return any_of(
- MF, [&P](const MachineBasicBlock &MBB) { return any_of(MBB, P); });
- };
- EnforceCallBoundary = AnyInstr(mayNeedNonDefaultMode) &&
- !AnyInstr([TII](const MachineInstr &MI) {
- return writesRoundMode(MI, TII);
- });
-
// Processing is performed in a number of phases
// Phase 1 - determine the initial mode required by each block, and add setreg
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.p1.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.p1.f16.ll
index 2c7cca06eb102..a5f38cafabb35 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.p1.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.p1.f16.ll
@@ -9,6 +9,8 @@ define amdgpu_ps float @interp_f16(float %i, i32 inreg %m0) #0 {
; GFX9-32BANK-NEXT: s_mov_b32 m0, s0
; GFX9-32BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
; GFX9-32BANK-NEXT: v_interp_p1ll_f16 v0, v0, attr2.y
+; GFX9-32BANK-NEXT: s_nop 0
+; GFX9-32BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GFX9-32BANK-NEXT: ; return to shader part epilog
;
; GFX8-32BANK-LABEL: interp_f16:
@@ -16,6 +18,8 @@ define amdgpu_ps float @interp_f16(float %i, i32 inreg %m0) #0 {
; GFX8-32BANK-NEXT: s_mov_b32 m0, s0
; GFX8-32BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
; GFX8-32BANK-NEXT: v_interp_p1ll_f16 v0, v0, attr2.y
+; GFX8-32BANK-NEXT: s_nop 0
+; GFX8-32BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GFX8-32BANK-NEXT: ; return to shader part epilog
;
; GFX8-16BANK-LABEL: interp_f16:
@@ -24,6 +28,8 @@ define amdgpu_ps float @interp_f16(float %i, i32 inreg %m0) #0 {
; GFX8-16BANK-NEXT: v_interp_mov_f32_e32 v1, p0, attr2.y
; GFX8-16BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
; GFX8-16BANK-NEXT: v_interp_p1lv_f16 v0, v0, attr2.y, v1
+; GFX8-16BANK-NEXT: s_nop 0
+; GFX8-16BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GFX8-16BANK-NEXT: ; return to shader part epilog
%res = call float @llvm.amdgcn.interp.p1.f16(float %i, i32 1, i32 2, i1 false, i32 %m0)
ret float %res
@@ -35,6 +41,8 @@ define amdgpu_ps float @interp_f16_high(float %i, i32 inreg %m0) #0 {
; GFX9-32BANK-NEXT: s_mov_b32 m0, s0
; GFX9-32BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
; GFX9-32BANK-NEXT: v_interp_p1ll_f16 v0, v0, attr2.y high
+; GFX9-32BANK-NEXT: s_nop 0
+; GFX9-32BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GFX9-32BANK-NEXT: ; return to shader part epilog
;
; GFX8-32BANK-LABEL: interp_f16_high:
@@ -42,6 +50,8 @@ define amdgpu_ps float @interp_f16_high(float %i, i32 inreg %m0) #0 {
; GFX8-32BANK-NEXT: s_mov_b32 m0, s0
; GFX8-32BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
; GFX8-32BANK-NEXT: v_interp_p1ll_f16 v0, v0, attr2.y high
+; GFX8-32BANK-NEXT: s_nop 0
+; GFX8-32BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GFX8-32BANK-NEXT: ; return to shader part epilog
;
; GFX8-16BANK-LABEL: interp_f16_high:
@@ -50,6 +60,8 @@ define amdgpu_ps float @interp_f16_high(float %i, i32 inreg %m0) #0 {
; GFX8-16BANK-NEXT: v_interp_mov_f32_e32 v1, p0, attr2.y
; GFX8-16BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
; GFX8-16BANK-NEXT: v_interp_p1lv_f16 v0, v0, attr2.y, v1 high
+; GFX8-16BANK-NEXT: s_nop 0
+; GFX8-16BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GFX8-16BANK-NEXT: ; return to shader part epilog
%res = call float @llvm.amdgcn.interp.p1.f16(float %i, i32 1, i32 2, i1 true, i32 %m0)
ret float %res
@@ -61,6 +73,8 @@ define amdgpu_ps float @interp_f16_0_0(float %i, i32 inreg %m0) #0 {
; GFX9-32BANK-NEXT: s_mov_b32 m0, s0
; GFX9-32BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
; GFX9-32BANK-NEXT: v_interp_p1ll_f16 v0, v0, attr0.x
+; GFX9-32BANK-NEXT: s_nop 0
+; GFX9-32BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GFX9-32BANK-NEXT: ; return to shader part epilog
;
; GFX8-32BANK-LABEL: interp_f16_0_0:
@@ -68,6 +82,8 @@ define amdgpu_ps float @interp_f16_0_0(float %i, i32 inreg %m0) #0 {
; GFX8-32BANK-NEXT: s_mov_b32 m0, s0
; GFX8-32BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
; GFX8-32BANK-NEXT: v_interp_p1ll_f16 v0, v0, attr0.x
+; GFX8-32BANK-NEXT: s_nop 0
+; GFX8-32BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GFX8-32BANK-NEXT: ; return to shader part epilog
;
; GFX8-16BANK-LABEL: interp_f16_0_0:
@@ -76,6 +92,8 @@ define amdgpu_ps float @interp_f16_0_0(float %i, i32 inreg %m0) #0 {
; GFX8-16BANK-NEXT: v_interp_mov_f32_e32 v1, p0, attr0.x
; GFX8-16BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
; GFX8-16BANK-NEXT: v_interp_p1lv_f16 v0, v0, attr0.x, v1
+; GFX8-16BANK-NEXT: s_nop 0
+; GFX8-16BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GFX8-16BANK-NEXT: ; return to shader part epilog
%res = call float @llvm.amdgcn.interp.p1.f16(float %i, i32 0, i32 0, i1 false, i32 %m0)
ret float %res
@@ -89,6 +107,8 @@ define amdgpu_ps float @interp_f16_sgpr_i(float inreg %i,i32 inreg %m0) #0 {
; GFX9-32BANK-NEXT: s_mov_b32 m0, s1
; GFX9-32BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
; GFX9-32BANK-NEXT: v_interp_p1ll_f16 v0, v0, attr2.y
+; GFX9-32BANK-NEXT: s_nop 0
+; GFX9-32BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GFX9-32BANK-NEXT: ; return to shader part epilog
;
; GFX8-32BANK-LABEL: interp_f16_sgpr_i:
@@ -97,6 +117,8 @@ define amdgpu_ps float @interp_f16_sgpr_i(float inreg %i,i32 inreg %m0) #0 {
; GFX8-32BANK-NEXT: s_mov_b32 m0, s1
; GFX8-32BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
; GFX8-32BANK-NEXT: v_interp_p1ll_f16 v0, v0, attr2.y
+; GFX8-32BANK-NEXT: s_nop 0
+; GFX8-32BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GFX8-32BANK-NEXT: ; return to shader part epilog
;
; GFX8-16BANK-LABEL: interp_f16_sgpr_i:
@@ -106,6 +128,8 @@ define amdgpu_ps float @interp_f16_sgpr_i(float inreg %i,i32 inreg %m0) #0 {
; GFX8-16BANK-NEXT: v_interp_mov_f32_e32 v1, p0, attr2.y
; GFX8-16BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
; GFX8-16BANK-NEXT: v_interp_p1lv_f16 v0, v0, attr2.y, v1
+; GFX8-16BANK-NEXT: s_nop 0
+; GFX8-16BANK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GFX8-16BANK-NEXT: ; return to shader part epilog
%res = call float @llvm.amdgcn.interp.p1.f16(float %i, i32 1, i32 2, i1 false, i32 %m0)
ret float %res
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
index 2ab76c99095f8..195619cc1c3f9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll
@@ -41,30 +41,35 @@ define amdgpu_gs half @v_fptrunc_round_f32_to_f16_upward(float %a) {
; SDAG: ; %bb.0:
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
; SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; SDAG-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-LABEL: v_fptrunc_round_f32_to_f16_upward:
; GFX11-SDAG: ; %bb.0:
; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
; GFX11-SDAG-NEXT: v_cvt_f16_f32_e64 v0.l, v0
+; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GFX11-SDAG-NEXT: ; return to shader part epilog
;
; GFX11-GISEL-LABEL: v_fptrunc_round_f32_to_f16_upward:
; GFX11-GISEL: ; %bb.0:
; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v0.l, v0
+; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: v_fptrunc_round_f32_to_f16_upward:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: v_fptrunc_round_f32_to_f16_upward:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 1
; GFX12-NEXT: v_cvt_f16_f32_e64 v0.l, v0
+; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 0
; GFX12-NEXT: ; return to shader part epilog
%res = call half @llvm.fptrunc.round.f16.f32(float %a, metadata !"round.upward")
ret half %res
@@ -75,30 +80,35 @@ define amdgpu_gs half @v_fptrunc_round_f32_to_f16_downward(float %a) {
; SDAG: ; %bb.0:
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 1
; SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; SDAG-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-LABEL: v_fptrunc_round_f32_to_f16_downward:
; GFX11-SDAG: ; %bb.0:
; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 1
; GFX11-SDAG-NEXT: v_cvt_f16_f32_e64 v0.l, v0
+; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GFX11-SDAG-NEXT: ; return to shader part epilog
;
; GFX11-GISEL-LABEL: v_fptrunc_round_f32_to_f16_downward:
; GFX11-GISEL: ; %bb.0:
; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 1
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v0.l, v0
+; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: v_fptrunc_round_f32_to_f16_downward:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 1
; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: v_fptrunc_round_f32_to_f16_downward:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 1
; GFX12-NEXT: v_cvt_f16_f32_e64 v0.l, v0
+; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 0
; GFX12-NEXT: ; return to shader part epilog
%res = call half @llvm.fptrunc.round.f16.f32(float %a, metadata !"round.downward")
ret half %res
@@ -499,6 +509,7 @@ define amdgpu_gs i32 @s_fptrunc_round_f32_to_f16_upward(float inreg %a, ptr addr
; SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
; SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; SDAG-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-LABEL: s_fptrunc_round_f32_to_f16_upward:
@@ -508,6 +519,7 @@ define amdgpu_gs i32 @s_fptrunc_round_f32_to_f16_upward(float inreg %a, ptr addr
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GFX11-SDAG-NEXT: ; return to shader part epilog
;
; GFX11-GISEL-LABEL: s_fptrunc_round_f32_to_f16_upward:
@@ -517,6 +529,7 @@ define amdgpu_gs i32 @s_fptrunc_round_f32_to_f16_upward(float inreg %a, ptr addr
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-GISEL-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: s_fptrunc_round_f32_to_f16_upward:
@@ -526,6 +539,7 @@ define amdgpu_gs i32 @s_fptrunc_round_f32_to_f16_upward(float inreg %a, ptr addr
; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GISEL-NEXT: s_and_b32 s0, s0, 0xffff
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: s_fptrunc_round_f32_to_f16_upward:
@@ -534,6 +548,7 @@ define amdgpu_gs i32 @s_fptrunc_round_f32_to_f16_upward(float inreg %a, ptr addr
; GFX12-NEXT: s_cvt_f16_f32 s0, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
; GFX12-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 0
; GFX12-NEXT: ; return to shader part epilog
%res = call half @llvm.fptrunc.round.f16.f32(float %a, metadata !"round.upward")
%bitcast = bitcast half %res to i16
@@ -549,6 +564,7 @@ define amdgpu_gs i32 @s_fptrunc_round_f32_to_f16_downward(float inreg %a, ptr ad
; SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
; SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; SDAG-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-LABEL: s_fptrunc_round_f32_to_f16_downward:
@@ -558,6 +574,7 @@ define amdgpu_gs i32 @s_fptrunc_round_f32_to_f16_downward(float inreg %a, ptr ad
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GFX11-SDAG-NEXT: ; return to shader part epilog
;
; GFX11-GISEL-LABEL: s_fptrunc_round_f32_to_f16_downward:
@@ -567,6 +584,7 @@ define amdgpu_gs i32 @s_fptrunc_round_f32_to_f16_downward(float inreg %a, ptr ad
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-GISEL-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: s_fptrunc_round_f32_to_f16_downward:
@@ -576,6 +594,7 @@ define amdgpu_gs i32 @s_fptrunc_round_f32_to_f16_downward(float inreg %a, ptr ad
; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GISEL-NEXT: s_and_b32 s0, s0, 0xffff
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: s_fptrunc_round_f32_to_f16_downward:
@@ -584,6 +603,7 @@ define amdgpu_gs i32 @s_fptrunc_round_f32_to_f16_downward(float inreg %a, ptr ad
; GFX12-NEXT: s_cvt_f16_f32 s0, s0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
; GFX12-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 0
; GFX12-NEXT: ; return to shader part epilog
%res = call half @llvm.fptrunc.round.f16.f32(float %a, metadata !"round.downward")
%bitcast = bitcast half %res to i16
@@ -723,6 +743,7 @@ define amdgpu_gs <2 x half> @v_fptrunc_round_v2f32_to_v2f16_upward(<2 x float> %
; SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
; SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
; SDAG-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; SDAG-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-LABEL: v_fptrunc_round_v2f32_to_v2f16_upward:
@@ -732,6 +753,7 @@ define amdgpu_gs <2 x half> @v_fptrunc_round_v2f32_to_v2f16_upward(<2 x float> %
; GFX11-SDAG-NEXT: v_cvt_f16_f32_e64 v1.l, v0
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, v1
+; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GFX11-SDAG-NEXT: ; return to shader part epilog
;
; GFX11-GISEL-LABEL: v_fptrunc_round_v2f32_to_v2f16_upward:
@@ -739,6 +761,7 @@ define amdgpu_gs <2 x half> @v_fptrunc_round_v2f32_to_v2f16_upward(<2 x float> %
; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v0.l, v0
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v0.h, v1
+; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: v_fptrunc_round_v2f32_to_v2f16_upward:
@@ -747,6 +770,7 @@ define amdgpu_gs <2 x half> @v_fptrunc_round_v2f32_to_v2f16_upward(<2 x float> %
; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
; GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: v_fptrunc_round_v2f32_to_v2f16_upward:
@@ -756,6 +780,7 @@ define amdgpu_gs <2 x half> @v_fptrunc_round_v2f32_to_v2f16_upward(<2 x float> %
; GFX12-SDAG-NEXT: v_cvt_f16_f32_e64 v1.l, v0
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, v1
+; GFX12-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 0
; GFX12-SDAG-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-LABEL: v_fptrunc_round_v2f32_to_v2f16_upward:
@@ -763,6 +788,7 @@ define amdgpu_gs <2 x half> @v_fptrunc_round_v2f32_to_v2f16_upward(<2 x float> %
; GFX12-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 1
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v0.l, v0
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v0.h, v1
+; GFX12-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 0
; GFX12-GISEL-NEXT: ; return to shader part epilog
%res = call <2 x half> @llvm.fptrunc.round.v2f16.v2f32(<2 x float> %a, metadata !"round.upward")
ret <2 x half> %res
@@ -775,6 +801,7 @@ define amdgpu_gs <2 x half> @v_fptrunc_round_v2f32_to_v2f16_downward(<2 x float>
; SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
; SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
; SDAG-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; SDAG-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-LABEL: v_fptrunc_round_v2f32_to_v2f16_downward:
@@ -784,6 +811,7 @@ define amdgpu_gs <2 x half> @v_fptrunc_round_v2f32_to_v2f16_downward(<2 x float>
; GFX11-SDAG-NEXT: v_cvt_f16_f32_e64 v1.l, v0
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, v1
+; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GFX11-SDAG-NEXT: ; return to shader part epilog
;
; GFX11-GISEL-LABEL: v_fptrunc_round_v2f32_to_v2f16_downward:
@@ -791,6 +819,7 @@ define amdgpu_gs <2 x half> @v_fptrunc_round_v2f32_to_v2f16_downward(<2 x float>
; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 1
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v0.l, v0
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v0.h, v1
+; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: v_fptrunc_round_v2f32_to_v2f16_downward:
@@ -799,6 +828,7 @@ define amdgpu_gs <2 x half> @v_fptrunc_round_v2f32_to_v2f16_downward(<2 x float>
; GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
; GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: v_fptrunc_round_v2f32_to_v2f16_downward:
@@ -808,6 +838,7 @@ define amdgpu_gs <2 x half> @v_fptrunc_round_v2f32_to_v2f16_downward(<2 x float>
; GFX12-SDAG-NEXT: v_cvt_f16_f32_e64 v1.l, v0
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, v1
+; GFX12-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 0
; GFX12-SDAG-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-LABEL: v_fptrunc_round_v2f32_to_v2f16_downward:
@@ -815,6 +846,7 @@ define amdgpu_gs <2 x half> @v_fptrunc_round_v2f32_to_v2f16_downward(<2 x float>
; GFX12-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 1
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v0.l, v0
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v0.h, v1
+; GFX12-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 0
; GFX12-GISEL-NEXT: ; return to shader part epilog
%res = call <2 x half> @llvm.fptrunc.round.v2f16.v2f32(<2 x float> %a, metadata !"round.downward")
ret <2 x half> %res
@@ -947,6 +979,7 @@ define amdgpu_gs <2 x i32> @s_fptrunc_round_v2f32_to_v2f16_upward(<2 x float> in
; SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SDAG-NEXT: v_readfirstlane_b32 s0, v0
; SDAG-NEXT: v_readfirstlane_b32 s1, v1
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; SDAG-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-LABEL: s_fptrunc_round_v2f32_to_v2f16_upward:
@@ -960,6 +993,7 @@ define amdgpu_gs <2 x i32> @s_fptrunc_round_v2f32_to_v2f16_upward(<2 x float> in
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-SDAG-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GFX11-SDAG-NEXT: ; return to shader part epilog
;
; GFX11-GISEL-LABEL: s_fptrunc_round_v2f32_to_v2f16_upward:
@@ -972,6 +1006,7 @@ define amdgpu_gs <2 x i32> @s_fptrunc_round_v2f32_to_v2f16_upward(<2 x float> in
; GFX11-GISEL-NEXT: v_readfirstlane_b32 s1, v1
; GFX11-GISEL-NEXT: s_and_b32 s0, 0xffff, s0
; GFX11-GISEL-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: s_fptrunc_round_v2f32_to_v2f16_upward:
@@ -986,6 +1021,7 @@ define amdgpu_gs <2 x i32> @s_fptrunc_round_v2f32_to_v2f16_upward(<2 x float> in
; GISEL-NEXT: s_pack_ll_b32_b16 s1, s0, s1
; GISEL-NEXT: s_and_b32 s0, s1, 0xffff
; GISEL-NEXT: s_lshr_b32 s1, s1, 16
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: s_fptrunc_round_v2f32_to_v2f16_upward:
@@ -996,6 +1032,7 @@ define amdgpu_gs <2 x i32> @s_fptrunc_round_v2f32_to_v2f16_upward(<2 x float> in
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_2) | instskip(NEXT) | instid1(SALU_CYCLE_2)
; GFX12-NEXT: s_and_b32 s0, 0xffff, s0
; GFX12-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 0
; GFX12-NEXT: ; return to shader part epilog
%res = call <2 x half> @llvm.fptrunc.round.v2f16.v2f32(<2 x float> %a, metadata !"round.upward")
%bitcast = bitcast <2 x half> %res to <2 x i16>
@@ -1015,6 +1052,7 @@ define amdgpu_gs <2 x i32> @s_fptrunc_round_v2f32_to_v2f16_downward(<2 x float>
; SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SDAG-NEXT: v_readfirstlane_b32 s0, v0
; SDAG-NEXT: v_readfirstlane_b32 s1, v1
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; SDAG-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-LABEL: s_fptrunc_round_v2f32_to_v2f16_downward:
@@ -1028,6 +1066,7 @@ define amdgpu_gs <2 x i32> @s_fptrunc_round_v2f32_to_v2f16_downward(<2 x float>
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-SDAG-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GFX11-SDAG-NEXT: ; return to shader part epilog
;
; GFX11-GISEL-LABEL: s_fptrunc_round_v2f32_to_v2f16_downward:
@@ -1040,6 +1079,7 @@ define amdgpu_gs <2 x i32> @s_fptrunc_round_v2f32_to_v2f16_downward(<2 x float>
; GFX11-GISEL-NEXT: v_readfirstlane_b32 s1, v1
; GFX11-GISEL-NEXT: s_and_b32 s0, 0xffff, s0
; GFX11-GISEL-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: s_fptrunc_round_v2f32_to_v2f16_downward:
@@ -1054,6 +1094,7 @@ define amdgpu_gs <2 x i32> @s_fptrunc_round_v2f32_to_v2f16_downward(<2 x float>
; GISEL-NEXT: s_pack_ll_b32_b16 s1, s0, s1
; GISEL-NEXT: s_and_b32 s0, s1, 0xffff
; GISEL-NEXT: s_lshr_b32 s1, s1, 16
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: s_fptrunc_round_v2f32_to_v2f16_downward:
@@ -1064,6 +1105,7 @@ define amdgpu_gs <2 x i32> @s_fptrunc_round_v2f32_to_v2f16_downward(<2 x float>
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_2) | instskip(NEXT) | instid1(SALU_CYCLE_2)
; GFX12-NEXT: s_and_b32 s0, 0xffff, s0
; GFX12-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 0
; GFX12-NEXT: ; return to shader part epilog
%res = call <2 x half> @llvm.fptrunc.round.v2f16.v2f32(<2 x float> %a, metadata !"round.downward")
%bitcast = bitcast <2 x half> %res to <2 x i16>
@@ -1293,6 +1335,7 @@ define amdgpu_gs <3 x half> @v_fptrunc_round_v3f32_to_v3f16_upward(<3 x float> %
; SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
; SDAG-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; SDAG-NEXT: v_cvt_f16_f32_e32 v1, v2
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; SDAG-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-LABEL: v_fptrunc_round_v3f32_to_v3f16_upward:
@@ -1303,6 +1346,7 @@ define amdgpu_gs <3 x half> @v_fptrunc_round_v3f32_to_v3f16_upward(<3 x float> %
; GFX11-SDAG-NEXT: v_cvt_f16_f32_e64 v1.l, v2
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, v3
+; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GFX11-SDAG-NEXT: ; return to shader part epilog
;
; GFX11-GISEL-LABEL: v_fptrunc_round_v3f32_to_v3f16_upward:
@@ -1311,6 +1355,7 @@ define amdgpu_gs <3 x half> @v_fptrunc_round_v3f32_to_v3f16_upward(<3 x float> %
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v0.l, v0
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v0.h, v1
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v1.l, v2
+; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: v_fptrunc_round_v3f32_to_v3f16_upward:
@@ -1322,6 +1367,7 @@ define amdgpu_gs <3 x half> @v_fptrunc_round_v3f32_to_v3f16_upward(<3 x float> %
; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
; GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
; GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: v_fptrunc_round_v3f32_to_v3f16_upward:
@@ -1332,6 +1378,7 @@ define amdgpu_gs <3 x half> @v_fptrunc_round_v3f32_to_v3f16_upward(<3 x float> %
; GFX12-SDAG-NEXT: v_cvt_f16_f32_e64 v1.l, v2
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, v3
+; GFX12-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 0
; GFX12-SDAG-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-LABEL: v_fptrunc_round_v3f32_to_v3f16_upward:
@@ -1340,6 +1387,7 @@ define amdgpu_gs <3 x half> @v_fptrunc_round_v3f32_to_v3f16_upward(<3 x float> %
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v0.l, v0
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v0.h, v1
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v1.l, v2
+; GFX12-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 0
; GFX12-GISEL-NEXT: ; return to shader part epilog
%res = call <3 x half> @llvm.fptrunc.round.v3f16.v3f32(<3 x float> %a, metadata !"round.upward")
ret <3 x half> %res
@@ -1353,6 +1401,7 @@ define amdgpu_gs <3 x half> @v_fptrunc_round_v3f32_to_v3f16_downward(<3 x float>
; SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
; SDAG-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; SDAG-NEXT: v_cvt_f16_f32_e32 v1, v2
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; SDAG-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-LABEL: v_fptrunc_round_v3f32_to_v3f16_downward:
@@ -1363,6 +1412,7 @@ define amdgpu_gs <3 x half> @v_fptrunc_round_v3f32_to_v3f16_downward(<3 x float>
; GFX11-SDAG-NEXT: v_cvt_f16_f32_e64 v1.l, v2
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, v3
+; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GFX11-SDAG-NEXT: ; return to shader part epilog
;
; GFX11-GISEL-LABEL: v_fptrunc_round_v3f32_to_v3f16_downward:
@@ -1371,6 +1421,7 @@ define amdgpu_gs <3 x half> @v_fptrunc_round_v3f32_to_v3f16_downward(<3 x float>
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v0.l, v0
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v0.h, v1
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v1.l, v2
+; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: v_fptrunc_round_v3f32_to_v3f16_downward:
@@ -1382,6 +1433,7 @@ define amdgpu_gs <3 x half> @v_fptrunc_round_v3f32_to_v3f16_downward(<3 x float>
; GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
; GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
; GISEL-NEXT: v_or_b32_sdwa v1, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: v_fptrunc_round_v3f32_to_v3f16_downward:
@@ -1392,6 +1444,7 @@ define amdgpu_gs <3 x half> @v_fptrunc_round_v3f32_to_v3f16_downward(<3 x float>
; GFX12-SDAG-NEXT: v_cvt_f16_f32_e64 v1.l, v2
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, v3
+; GFX12-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 0
; GFX12-SDAG-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-LABEL: v_fptrunc_round_v3f32_to_v3f16_downward:
@@ -1400,6 +1453,7 @@ define amdgpu_gs <3 x half> @v_fptrunc_round_v3f32_to_v3f16_downward(<3 x float>
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v0.l, v0
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v0.h, v1
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v1.l, v2
+; GFX12-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 0
; GFX12-GISEL-NEXT: ; return to shader part epilog
%res = call <3 x half> @llvm.fptrunc.round.v3f16.v3f32(<3 x float> %a, metadata !"round.downward")
ret <3 x half> %res
@@ -1415,6 +1469,7 @@ define amdgpu_gs <4 x half> @v_fptrunc_round_v4f32_to_v4f16_upward(<4 x float> %
; SDAG-NEXT: v_cvt_f16_f32_e32 v3, v3
; SDAG-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; SDAG-NEXT: v_perm_b32 v1, v3, v2, 0x5040100
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; SDAG-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-LABEL: v_fptrunc_round_v4f32_to_v4f16_upward:
@@ -1426,6 +1481,7 @@ define amdgpu_gs <4 x half> @v_fptrunc_round_v4f32_to_v4f16_upward(<4 x float> %
; GFX11-SDAG-NEXT: v_cvt_f16_f32_e64 v3.l, v2
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v3
+; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GFX11-SDAG-NEXT: ; return to shader part epilog
;
; GFX11-GISEL-LABEL: v_fptrunc_round_v4f32_to_v4f16_upward:
@@ -1435,6 +1491,7 @@ define amdgpu_gs <4 x half> @v_fptrunc_round_v4f32_to_v4f16_upward(<4 x float> %
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v0.h, v1
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v1.l, v2
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v1.h, v3
+; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: v_fptrunc_round_v4f32_to_v4f16_upward:
@@ -1446,6 +1503,7 @@ define amdgpu_gs <4 x half> @v_fptrunc_round_v4f32_to_v4f16_upward(<4 x float> %
; GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
; GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
; GISEL-NEXT: v_pack_b32_f16 v1, v2, v3
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: v_fptrunc_round_v4f32_to_v4f16_upward:
@@ -1457,6 +1515,7 @@ define amdgpu_gs <4 x half> @v_fptrunc_round_v4f32_to_v4f16_upward(<4 x float> %
; GFX12-SDAG-NEXT: v_cvt_f16_f32_e64 v3.l, v2
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v3
+; GFX12-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 0
; GFX12-SDAG-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-LABEL: v_fptrunc_round_v4f32_to_v4f16_upward:
@@ -1466,6 +1525,7 @@ define amdgpu_gs <4 x half> @v_fptrunc_round_v4f32_to_v4f16_upward(<4 x float> %
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v0.h, v1
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v1.l, v2
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v1.h, v3
+; GFX12-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 0
; GFX12-GISEL-NEXT: ; return to shader part epilog
%res = call <4 x half> @llvm.fptrunc.round.v4f16.v4f32(<4 x float> %a, metadata !"round.upward")
ret <4 x half> %res
@@ -1481,6 +1541,7 @@ define amdgpu_gs <4 x half> @v_fptrunc_round_v4f32_to_v4f16_downward(<4 x float>
; SDAG-NEXT: v_cvt_f16_f32_e32 v3, v3
; SDAG-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; SDAG-NEXT: v_perm_b32 v1, v3, v2, 0x5040100
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; SDAG-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-LABEL: v_fptrunc_round_v4f32_to_v4f16_downward:
@@ -1492,6 +1553,7 @@ define amdgpu_gs <4 x half> @v_fptrunc_round_v4f32_to_v4f16_downward(<4 x float>
; GFX11-SDAG-NEXT: v_cvt_f16_f32_e64 v3.l, v2
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v3
+; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GFX11-SDAG-NEXT: ; return to shader part epilog
;
; GFX11-GISEL-LABEL: v_fptrunc_round_v4f32_to_v4f16_downward:
@@ -1501,6 +1563,7 @@ define amdgpu_gs <4 x half> @v_fptrunc_round_v4f32_to_v4f16_downward(<4 x float>
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v0.h, v1
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v1.l, v2
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v1.h, v3
+; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: v_fptrunc_round_v4f32_to_v4f16_downward:
@@ -1512,6 +1575,7 @@ define amdgpu_gs <4 x half> @v_fptrunc_round_v4f32_to_v4f16_downward(<4 x float>
; GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
; GISEL-NEXT: v_pack_b32_f16 v0, v0, v1
; GISEL-NEXT: v_pack_b32_f16 v1, v2, v3
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: v_fptrunc_round_v4f32_to_v4f16_downward:
@@ -1523,6 +1587,7 @@ define amdgpu_gs <4 x half> @v_fptrunc_round_v4f32_to_v4f16_downward(<4 x float>
; GFX12-SDAG-NEXT: v_cvt_f16_f32_e64 v3.l, v2
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v3
+; GFX12-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 0
; GFX12-SDAG-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-LABEL: v_fptrunc_round_v4f32_to_v4f16_downward:
@@ -1532,6 +1597,7 @@ define amdgpu_gs <4 x half> @v_fptrunc_round_v4f32_to_v4f16_downward(<4 x float>
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v0.h, v1
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v1.l, v2
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v1.h, v3
+; GFX12-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 0
; GFX12-GISEL-NEXT: ; return to shader part epilog
%res = call <4 x half> @llvm.fptrunc.round.v4f16.v4f32(<4 x float> %a, metadata !"round.downward")
ret <4 x half> %res
@@ -1553,6 +1619,7 @@ define amdgpu_gs <8 x half> @v_fptrunc_round_v8f32_to_v8f16_upward(<8 x float> %
; SDAG-NEXT: v_perm_b32 v1, v3, v2, 0x5040100
; SDAG-NEXT: v_perm_b32 v2, v5, v4, 0x5040100
; SDAG-NEXT: v_perm_b32 v3, v7, v6, 0x5040100
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; SDAG-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-LABEL: v_fptrunc_round_v8f32_to_v8f16_upward:
@@ -1569,6 +1636,7 @@ define amdgpu_gs <8 x half> @v_fptrunc_round_v8f32_to_v8f16_upward(<8 x float> %
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v3
; GFX11-SDAG-NEXT: v_dual_mov_b32 v2, v5 :: v_dual_mov_b32 v3, v7
+; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GFX11-SDAG-NEXT: ; return to shader part epilog
;
; GFX11-GISEL-LABEL: v_fptrunc_round_v8f32_to_v8f16_upward:
@@ -1582,6 +1650,7 @@ define amdgpu_gs <8 x half> @v_fptrunc_round_v8f32_to_v8f16_upward(<8 x float> %
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v2.h, v5
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v3.l, v6
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v3.h, v7
+; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: v_fptrunc_round_v8f32_to_v8f16_upward:
@@ -1599,6 +1668,7 @@ define amdgpu_gs <8 x half> @v_fptrunc_round_v8f32_to_v8f16_upward(<8 x float> %
; GISEL-NEXT: v_pack_b32_f16 v1, v2, v3
; GISEL-NEXT: v_pack_b32_f16 v2, v4, v5
; GISEL-NEXT: v_pack_b32_f16 v3, v6, v7
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: v_fptrunc_round_v8f32_to_v8f16_upward:
@@ -1615,6 +1685,7 @@ define amdgpu_gs <8 x half> @v_fptrunc_round_v8f32_to_v8f16_upward(<8 x float> %
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v3
; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, v5 :: v_dual_mov_b32 v3, v7
+; GFX12-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 0
; GFX12-SDAG-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-LABEL: v_fptrunc_round_v8f32_to_v8f16_upward:
@@ -1628,6 +1699,7 @@ define amdgpu_gs <8 x half> @v_fptrunc_round_v8f32_to_v8f16_upward(<8 x float> %
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v2.h, v5
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v3.l, v6
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v3.h, v7
+; GFX12-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 0
; GFX12-GISEL-NEXT: ; return to shader part epilog
%res = call <8 x half> @llvm.fptrunc.round.v8f16.v8f32(<8 x float> %a, metadata !"round.upward")
ret <8 x half> %res
@@ -1649,6 +1721,7 @@ define amdgpu_gs <8 x half> @v_fptrunc_round_v8f32_to_v8f16_downward(<8 x float>
; SDAG-NEXT: v_perm_b32 v1, v3, v2, 0x5040100
; SDAG-NEXT: v_perm_b32 v2, v5, v4, 0x5040100
; SDAG-NEXT: v_perm_b32 v3, v7, v6, 0x5040100
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; SDAG-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-LABEL: v_fptrunc_round_v8f32_to_v8f16_downward:
@@ -1665,6 +1738,7 @@ define amdgpu_gs <8 x half> @v_fptrunc_round_v8f32_to_v8f16_downward(<8 x float>
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v3
; GFX11-SDAG-NEXT: v_dual_mov_b32 v2, v5 :: v_dual_mov_b32 v3, v7
+; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GFX11-SDAG-NEXT: ; return to shader part epilog
;
; GFX11-GISEL-LABEL: v_fptrunc_round_v8f32_to_v8f16_downward:
@@ -1678,6 +1752,7 @@ define amdgpu_gs <8 x half> @v_fptrunc_round_v8f32_to_v8f16_downward(<8 x float>
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v2.h, v5
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v3.l, v6
; GFX11-GISEL-NEXT: v_cvt_f16_f32_e64 v3.h, v7
+; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: v_fptrunc_round_v8f32_to_v8f16_downward:
@@ -1695,6 +1770,7 @@ define amdgpu_gs <8 x half> @v_fptrunc_round_v8f32_to_v8f16_downward(<8 x float>
; GISEL-NEXT: v_pack_b32_f16 v1, v2, v3
; GISEL-NEXT: v_pack_b32_f16 v2, v4, v5
; GISEL-NEXT: v_pack_b32_f16 v3, v6, v7
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: v_fptrunc_round_v8f32_to_v8f16_downward:
@@ -1711,6 +1787,7 @@ define amdgpu_gs <8 x half> @v_fptrunc_round_v8f32_to_v8f16_downward(<8 x float>
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v3
; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, v5 :: v_dual_mov_b32 v3, v7
+; GFX12-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 0
; GFX12-SDAG-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-LABEL: v_fptrunc_round_v8f32_to_v8f16_downward:
@@ -1724,6 +1801,7 @@ define amdgpu_gs <8 x half> @v_fptrunc_round_v8f32_to_v8f16_downward(<8 x float>
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v2.h, v5
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v3.l, v6
; GFX12-GISEL-NEXT: v_cvt_f16_f32_e64 v3.h, v7
+; GFX12-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 0
; GFX12-GISEL-NEXT: ; return to shader part epilog
%res = call <8 x half> @llvm.fptrunc.round.v8f16.v8f32(<8 x float> %a, metadata !"round.downward")
ret <8 x half> %res
@@ -1748,12 +1826,14 @@ define amdgpu_gs float @v_fptrunc_round_f64_to_f32_upward(double %a) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; CHECK-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: v_fptrunc_round_f64_to_f32_upward:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 1
; GFX12-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 0
; GFX12-NEXT: ; return to shader part epilog
%res = call float @llvm.fptrunc.round.f32.f64(double %a, metadata !"round.upward")
ret float %res
@@ -1764,12 +1844,14 @@ define amdgpu_gs float @v_fptrunc_round_f64_to_f32_downward(double %a) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 1
; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; CHECK-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: v_fptrunc_round_f64_to_f32_downward:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 1
; GFX12-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 0
; GFX12-NEXT: ; return to shader part epilog
%res = call float @llvm.fptrunc.round.f32.f64(double %a, metadata !"round.downward")
ret float %res
@@ -1780,12 +1862,14 @@ define amdgpu_gs float @v_fptrunc_round_f64_to_f32_towardzero(double %a) {
; CHECK: ; %bb.0:
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; CHECK-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: v_fptrunc_round_f64_to_f32_towardzero:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 3
; GFX12-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0
; GFX12-NEXT: ; return to shader part epilog
%res = call float @llvm.fptrunc.round.f32.f64(double %a, metadata !"round.towardzero")
ret float %res
@@ -1798,6 +1882,7 @@ define amdgpu_gs float @s_fptrunc_round_f64_to_f32_upward(double inreg %a) {
; SDAG-NEXT: v_mov_b32_e32 v1, s1
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
; SDAG-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; SDAG-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-LABEL: s_fptrunc_round_f64_to_f32_upward:
@@ -1806,6 +1891,7 @@ define amdgpu_gs float @s_fptrunc_round_f64_to_f32_upward(double inreg %a) {
; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GFX11-SDAG-NEXT: ; return to shader part epilog
;
; GFX11-GISEL-LABEL: s_fptrunc_round_f64_to_f32_upward:
@@ -1814,6 +1900,7 @@ define amdgpu_gs float @s_fptrunc_round_f64_to_f32_upward(double inreg %a) {
; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: s_fptrunc_round_f64_to_f32_upward:
@@ -1822,6 +1909,7 @@ define amdgpu_gs float @s_fptrunc_round_f64_to_f32_upward(double inreg %a) {
; GISEL-NEXT: v_mov_b32_e32 v1, s1
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
; GISEL-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: s_fptrunc_round_f64_to_f32_upward:
@@ -1830,6 +1918,7 @@ define amdgpu_gs float @s_fptrunc_round_f64_to_f32_upward(double inreg %a) {
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 1
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 1), 0
; GFX12-NEXT: ; return to shader part epilog
%res = call float @llvm.fptrunc.round.f32.f64(double %a, metadata !"round.upward")
ret float %res
@@ -1842,6 +1931,7 @@ define amdgpu_gs float @s_fptrunc_round_f64_to_f32_downward(double inreg %a) {
; SDAG-NEXT: v_mov_b32_e32 v1, s1
; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 1
; SDAG-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; SDAG-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-LABEL: s_fptrunc_round_f64_to_f32_downward:
@@ -1850,6 +1940,7 @@ define amdgpu_gs float @s_fptrunc_round_f64_to_f32_downward(double inreg %a) {
; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX11-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GFX11-SDAG-NEXT: ; return to shader part epilog
;
; GFX11-GISEL-LABEL: s_fptrunc_round_f64_to_f32_downward:
@@ -1858,6 +1949,7 @@ define amdgpu_gs float @s_fptrunc_round_f64_to_f32_downward(double inreg %a) {
; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX11-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GFX11-GISEL-NEXT: ; return to shader part epilog
;
; GISEL-LABEL: s_fptrunc_round_f64_to_f32_downward:
@@ -1866,6 +1958,7 @@ define amdgpu_gs float @s_fptrunc_round_f64_to_f32_downward(double inreg %a) {
; GISEL-NEXT: v_mov_b32_e32 v1, s1
; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 1
; GISEL-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 3, 1), 0
; GISEL-NEXT: ; return to shader part epilog
;
; GFX12-LABEL: s_fptrunc_round_f64_to_f32_downward:
@@ -1874,6 +1967,7 @@ define amdgpu_gs float @s_fptrunc_round_f64_to_f32_downward(double inreg %a) {
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 1
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 3, 1), 0
; GFX12-NEXT: ; return to shader part epilog
%res = call float @llvm.fptrunc.round.f32.f64(double %a, metadata !"round.downward")
ret float %res
diff --git a/llvm/test/CodeGen/AMDGPU/mode-register-call-boundary.ll b/llvm/test/CodeGen/AMDGPU/mode-register-call-boundary.ll
index 5b91b691a74b1..0390dc9183627 100644
--- a/llvm/test/CodeGen/AMDGPU/mode-register-call-boundary.ll
+++ b/llvm/test/CodeGen/AMDGPU/mode-register-call-boundary.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck %s
+; RUN: llc -mtriple=amdgpu9.00 < %s | FileCheck %s
-; A mode SIModeRegister set for llvm.fptrunc.round is restored at a call or a
-; return. A mode the program asked for is not.
+; SIModeRegister restores the default mode at a call or a return, unless the
+; program asked for the mode itself.
declare void @sink(float)
declare float @tail_target(float)
@@ -14,7 +14,7 @@ define float @ret_restores_default(double %a) {
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; CHECK-NEXT: s_setpc_b64 s[30:31]
%t = call float @llvm.fptrunc.round.f32.f64(double %a, metadata !"round.upward")
ret float %t
@@ -31,7 +31,7 @@ define float @tail_call_restores_default(double %a) {
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; CHECK-NEXT: s_setpc_b64 s[16:17]
%t = call float @llvm.fptrunc.round.f32.f64(double %a, metadata !"round.upward")
%u = tail call float @tail_target(float %t)
@@ -58,8 +58,10 @@ define void @call_restores_default(double %a) {
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; CHECK-NEXT: s_swappc_b64 s[30:31], s[16:17]
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
; CHECK-NEXT: v_readlane_b32 s30, v40, 0
; CHECK-NEXT: v_readlane_b32 s31, v40, 1
; CHECK-NEXT: s_mov_b32 s32, s33
@@ -69,6 +71,7 @@ define void @call_restores_default(double %a) {
; CHECK-NEXT: s_mov_b64 exec, s[6:7]
; CHECK-NEXT: s_mov_b32 s33, s4
; CHECK-NEXT: s_waitcnt vmcnt(0)
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; CHECK-NEXT: s_setpc_b64 s[30:31]
%t = call float @llvm.fptrunc.round.f32.f64(double %a, metadata !"round.upward")
call void @sink(float %t)
@@ -93,18 +96,19 @@ define amdgpu_kernel void @kernel_no_restore(ptr addrspace(1) %out, double %a) {
ret void
}
-; A return to the shader epilog is not a call boundary either.
-define amdgpu_gs float @gs_no_restore(double %a) {
-; CHECK-LABEL: gs_no_restore:
+; A return to the shader epilog is still a return to external code.
+define amdgpu_gs float @gs_restores_default(double %a) {
+; CHECK-LABEL: gs_restores_default:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 1
; CHECK-NEXT: v_cvt_f32_f64_e32 v0, v[0:1]
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 1), 0
; CHECK-NEXT: ; return to shader part epilog
%t = call float @llvm.fptrunc.round.f32.f64(double %a, metadata !"round.upward")
ret float %t
}
-; An explicit setreg changes the mode for the caller, so no restore.
define float @explicit_setreg_persists_with_fptrunc_round(double %a) {
; CHECK-LABEL: explicit_setreg_persists_with_fptrunc_round:
; CHECK: ; %bb.0:
>From 250a34656cbc6f2c00cc2caf117d654afd4ec887 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Thu, 10 Sep 2026 16:56:00 +0200
Subject: [PATCH 3/3] MI.isReturn
---
llvm/lib/Target/AMDGPU/SIModeRegister.cpp | 15 +--------------
1 file changed, 1 insertion(+), 14 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIModeRegister.cpp b/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
index ac17b8d607505..c49b8039edaf8 100644
--- a/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
+++ b/llvm/lib/Target/AMDGPU/SIModeRegister.cpp
@@ -221,19 +221,6 @@ getModeSetregField(const MachineInstr &MI, const SIInstrInfo *TII) {
return std::make_pair(Offset, maskTrailingOnes<unsigned>(Width) << Offset);
}
-// Not the wave-ending opcodes. Tail calls: see isCall.
-static bool isReturnToCaller(const MachineInstr &MI) {
- switch (MI.getOpcode()) {
- case AMDGPU::SI_RETURN:
- case AMDGPU::SI_RETURN_TO_EPILOG:
- case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN:
- case AMDGPU::S_SETPC_B64_return:
- return true;
- default:
- return false;
- }
-}
-
// Determine the Mode register setting required for this instruction.
// Instructions which don't use the Mode register return a null Status.
// Note this currently only deals with instructions that use the floating point
@@ -370,7 +357,7 @@ void SIModeRegister::processBlockPhase1(MachineBasicBlock &MBB,
} else {
NewInfo->Change = NewInfo->Change.mergeUnknown(Mask);
}
- } else if (MI.isCall() || isReturnToCaller(MI)) {
+ } else if (MI.isCall() || MI.isReturn()) {
// Whether a restore is needed is a whole function property, so only
// record the site here. Change is unaffected: Phase 3 pairs any restore
// it inserts with a re-set after the call.
More information about the llvm-commits
mailing list