[llvm] [AMDGPU][GlobalISel] Guard against use of `hi16` for SGPR registers (PR #188781)
Igor Wodiany via llvm-commits
llvm-commits at lists.llvm.org
Thu Mar 26 09:57:01 PDT 2026
https://github.com/IgWod updated https://github.com/llvm/llvm-project/pull/188781
>From 3053992ae6bce67990a846b9ba8780d5562490c4 Mon Sep 17 00:00:00 2001
From: Igor Wodiany <igor.wodiany at amd.com>
Date: Thu, 26 Mar 2026 14:57:05 +0000
Subject: [PATCH 1/2] [AMDGPU][GlobalISel] Guard aginst use of `hi16` for SGPR
registers
This patch makes several interconnected changes that need to be
merged together:
1) It updates redundant copies elimination after isel so it accounts
for sub-registers. This prevents `lo16` and `hi16` being treated as
the same values.
2) It updates machine verification to prevent use of `hi16` with SGPR
registers as it is not supported in pipelines following the isel.
3) Update `selectG_UNMERGE_VALUES` so instead of generating `hi16` for
SGPR it shifts higher bits into the destination register.
Tests:
1) `fto[s/u]i-sat-vector.ll`: The correct number of conversions is now generated
as higher 16-bits are handled correctly; however, it introduces
`lshr` instructions. This should be resolved in 1A8A8287 by ennabling
`s_cvt_hi_`.
2) `llvm.amdgcn.intersect_ray.ll`: GlobalISel generates a correct number
of pack instructions now; however the codegen is sub-optimal as it
should replace `lshr` instructions with `s_pack_hh_`. (future work)
3) `peephole-fold-imm.mir`: This test was explicitly using `hi16` so I
removed offending tests.
---
.../CodeGen/GlobalISel/InstructionSelect.cpp | 17 +-
.../AMDGPU/AMDGPUInstructionSelector.cpp | 10 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 29 ++-
llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll | 177 +++++++++++++-----
llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll | 157 ++++++++++++----
.../AMDGPU/llvm.amdgcn.intersect_ray.ll | 14 +-
.../test/CodeGen/AMDGPU/peephole-fold-imm.mir | 43 -----
7 files changed, 304 insertions(+), 143 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
index 5fb31a04e5fd0..4b629534caa6c 100644
--- a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
@@ -243,13 +243,16 @@ bool InstructionSelect::selectMachineFunction(MachineFunction &MF) {
continue;
Register SrcReg = MI.getOperand(1).getReg();
Register DstReg = MI.getOperand(0).getReg();
- if (SrcReg.isVirtual() && DstReg.isVirtual()) {
- auto SrcRC = MRI.getRegClass(SrcReg);
- auto DstRC = MRI.getRegClass(DstReg);
- if (SrcRC == DstRC) {
- MRI.replaceRegWith(DstReg, SrcReg);
- MI.eraseFromParent();
- }
+ unsigned SrcSubIdx = MI.getOperand(1).getSubReg();
+ unsigned DstSubIdx = MI.getOperand(0).getSubReg();
+ if (!SrcReg.isVirtual() || !DstReg.isVirtual() || SrcSubIdx != DstSubIdx)
+ continue;
+
+ const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);
+ const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
+ if (SrcRC == DstRC) {
+ MRI.replaceRegWith(DstReg, SrcReg);
+ MI.eraseFromParent();
}
}
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index f5747488225c5..cd0a78660dac0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -730,8 +730,14 @@ bool AMDGPUInstructionSelector::selectG_UNMERGE_VALUES(MachineInstr &MI) const {
ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(SrcRC, DstSize / 8);
for (int I = 0, E = NumDst; I != E; ++I) {
MachineOperand &Dst = MI.getOperand(I);
- BuildMI(*BB, &MI, DL, TII.get(TargetOpcode::COPY), Dst.getReg())
- .addReg(SrcReg, {}, SubRegs[I]);
+ // hi16:sreg_32 is not allowed so explicitly shift upper 16-bits.
+ if (SrcBank->getID() == AMDGPU::SGPRRegBankID && SubRegs[I] == AMDGPU::hi16)
+ BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_LSHR_B32), Dst.getReg())
+ .addReg(SrcReg)
+ .addImm(16);
+ else
+ BuildMI(*BB, &MI, DL, TII.get(TargetOpcode::COPY), Dst.getReg())
+ .addReg(SrcReg, {}, SubRegs[I]);
// Make sure the subregister index is valid for the source register.
SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubRegs[I]);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index df2700d414893..254dab030f25b 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5163,6 +5163,19 @@ static bool isSubRegOf(const SIRegisterInfo &TRI,
SubReg.getReg() == SuperVec.getReg();
}
+/// Check if given MachineOperand is an SGPR with SubReg hi16.
+static bool isSGPRHi16(const MachineRegisterInfo &MRI, const SIRegisterInfo &RI,
+ const MachineOperand &MO) {
+ if (!MO.isReg())
+ return false;
+
+ Register Reg = MO.getReg();
+ if (!Reg || !Reg.isVirtual() || !MRI.getRegClassOrNull(Reg))
+ return false;
+
+ return RI.isSGPRReg(MRI, Reg) && MO.getSubReg() == AMDGPU::hi16;
+}
+
// Verify the illegal copy from vector register to SGPR for generic opcode COPY
bool SIInstrInfo::verifyCopy(const MachineInstr &MI,
const MachineRegisterInfo &MRI,
@@ -5192,7 +5205,20 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
if (!MRI.isSSA() && MI.isCopy())
return verifyCopy(MI, MRI, ErrInfo);
- if (SIInstrInfo::isGenericOpcode(Opcode))
+ if (SIInstrInfo::isGenericOpcode(Opcode) && !MI.isCopy())
+ return true;
+
+ // Verify that none of the operands in either AMDGPU instruction or COPY
+ // is SGPR hi16.
+ const MCInstrDesc &Desc = get(Opcode);
+ for (int I = 0, E = Desc.getNumOperands(); I != E; ++I) {
+ if (isSGPRHi16(MRI, RI, MI.getOperand(I))) {
+ ErrInfo = "SGPR cannot use hi16 subreg";
+ return false;
+ }
+ }
+
+ if (MI.isCopy())
return true;
int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
@@ -5208,7 +5234,6 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
}
// Make sure the number of operands is correct.
- const MCInstrDesc &Desc = get(Opcode);
if (!Desc.isVariadic() &&
Desc.getNumOperands() != MI.getNumExplicitOperands()) {
ErrInfo = "Instruction has wrong number of operands.";
diff --git a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
index 590c2c748bfdc..6063bafe48891 100644
--- a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
@@ -3348,20 +3348,31 @@ define <4 x i1> @test_s_signed_v4f16_v4i1(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
-; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
; GFX12-GI-NEXT: s_cvt_i32_f32 s0, s0
-; GFX12-GI-NEXT: s_cvt_i32_f32 s1, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_i32_f32 s2, s2
+; GFX12-GI-NEXT: s_cvt_i32_f32 s1, s1
+; GFX12-GI-NEXT: s_cvt_i32_f32 s3, s3
; GFX12-GI-NEXT: s_min_i32 s0, s0, 0
-; GFX12-GI-NEXT: s_min_i32 s1, s1, 0
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s2, s2, 0
+; GFX12-GI-NEXT: s_min_i32 s1, s1, 0
+; GFX12-GI-NEXT: s_min_i32 s3, s3, 0
; GFX12-GI-NEXT: s_max_i32 s0, s0, -1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_max_i32 s2, s2, -1
; GFX12-GI-NEXT: s_max_i32 s1, s1, -1
+; GFX12-GI-NEXT: s_max_i32 s3, s3, -1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s0
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s1
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i1> @llvm.fptosi.sat.v4f16.v4i1(<4 x half> %f)
ret <4 x i1> %x
@@ -3597,20 +3608,31 @@ define <4 x i8> @test_s_signed_v4f16_v4i8(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
-; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
; GFX12-GI-NEXT: s_cvt_i32_f32 s0, s0
-; GFX12-GI-NEXT: s_cvt_i32_f32 s1, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_i32_f32 s2, s2
+; GFX12-GI-NEXT: s_cvt_i32_f32 s1, s1
+; GFX12-GI-NEXT: s_cvt_i32_f32 s3, s3
; GFX12-GI-NEXT: s_min_i32 s0, s0, 0x7f
-; GFX12-GI-NEXT: s_min_i32 s1, s1, 0x7f
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s2, s2, 0x7f
+; GFX12-GI-NEXT: s_min_i32 s1, s1, 0x7f
+; GFX12-GI-NEXT: s_min_i32 s3, s3, 0x7f
; GFX12-GI-NEXT: s_max_i32 s0, s0, 0xffffff80
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_max_i32 s2, s2, 0xffffff80
; GFX12-GI-NEXT: s_max_i32 s1, s1, 0xffffff80
+; GFX12-GI-NEXT: s_max_i32 s3, s3, 0xffffff80
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s0
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s1
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i8> @llvm.fptosi.sat.v4f16.v4i8(<4 x half> %f)
ret <4 x i8> %x
@@ -3757,10 +3779,13 @@ define <4 x i16> @test_s_signed_v4f16_v4i16(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.h, s2
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.l, s1
-; GFX12-GI-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX12-GI-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.h, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i16> @llvm.fptosi.sat.v4f16.v4i16(<4 x half> %f)
ret <4 x i16> %x
@@ -3878,19 +3903,23 @@ define <4 x i64> @test_s_signed_v4f16_v4i64(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
-; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
-; GFX12-GI-NEXT: v_mov_b32_e32 v5, 0
-; GFX12-GI-NEXT: v_mov_b32_e32 v7, 0
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
; GFX12-GI-NEXT: s_cvt_i32_f32 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_i32_f32 s2, s2
; GFX12-GI-NEXT: s_cvt_i32_f32 s1, s1
-; GFX12-GI-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-GI-NEXT: s_cvt_i32_f32 s3, s3
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s0
-; GFX12-GI-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GI-NEXT: v_mov_b32_e32 v4, s1
-; GFX12-GI-NEXT: v_mov_b32_e32 v6, s1
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, 0
+; GFX12-GI-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, 0
+; GFX12-GI-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, 0
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i64> @llvm.fptosi.sat.v4f16.v4i64(<4 x half> %f)
ret <4 x i64> %x
@@ -5463,30 +5492,51 @@ define <8 x i1> @test_s_signed_v8f16_v8i1(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s4, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s5, s1, 16
+; GFX12-GI-NEXT: s_lshr_b32 s6, s2, 16
+; GFX12-GI-NEXT: s_lshr_b32 s7, s3, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s4, s4
; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s5, s5
; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s6, s6
; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s7, s7
; GFX12-GI-NEXT: s_cvt_i32_f32 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_i32_f32 s4, s4
; GFX12-GI-NEXT: s_cvt_i32_f32 s1, s1
+; GFX12-GI-NEXT: s_cvt_i32_f32 s5, s5
; GFX12-GI-NEXT: s_cvt_i32_f32 s2, s2
+; GFX12-GI-NEXT: s_cvt_i32_f32 s6, s6
; GFX12-GI-NEXT: s_cvt_i32_f32 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_i32_f32 s7, s7
; GFX12-GI-NEXT: s_min_i32 s0, s0, 0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s4, s4, 0
; GFX12-GI-NEXT: s_min_i32 s1, s1, 0
+; GFX12-GI-NEXT: s_min_i32 s5, s5, 0
; GFX12-GI-NEXT: s_min_i32 s2, s2, 0
+; GFX12-GI-NEXT: s_min_i32 s6, s6, 0
; GFX12-GI-NEXT: s_min_i32 s3, s3, 0
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s7, s7, 0
; GFX12-GI-NEXT: s_max_i32 s0, s0, -1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_max_i32 s4, s4, -1
; GFX12-GI-NEXT: s_max_i32 s1, s1, -1
+; GFX12-GI-NEXT: s_max_i32 s5, s5, -1
; GFX12-GI-NEXT: s_max_i32 s2, s2, -1
+; GFX12-GI-NEXT: s_max_i32 s6, s6, -1
; GFX12-GI-NEXT: s_max_i32 s3, s3, -1
+; GFX12-GI-NEXT: s_max_i32 s7, s7, -1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s0
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s1
-; GFX12-GI-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s2
-; GFX12-GI-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s3
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s5
+; GFX12-GI-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s6
+; GFX12-GI-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i1> @llvm.fptosi.sat.v8f16.v8i1(<8 x half> %f)
ret <8 x i1> %x
@@ -5891,30 +5941,51 @@ define <8 x i8> @test_s_signed_v8f16_v8i8(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s4, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s5, s1, 16
+; GFX12-GI-NEXT: s_lshr_b32 s6, s2, 16
+; GFX12-GI-NEXT: s_lshr_b32 s7, s3, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s4, s4
; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s5, s5
; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s6, s6
; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s7, s7
; GFX12-GI-NEXT: s_cvt_i32_f32 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_i32_f32 s4, s4
; GFX12-GI-NEXT: s_cvt_i32_f32 s1, s1
+; GFX12-GI-NEXT: s_cvt_i32_f32 s5, s5
; GFX12-GI-NEXT: s_cvt_i32_f32 s2, s2
+; GFX12-GI-NEXT: s_cvt_i32_f32 s6, s6
; GFX12-GI-NEXT: s_cvt_i32_f32 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_i32_f32 s7, s7
; GFX12-GI-NEXT: s_min_i32 s0, s0, 0x7f
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s4, s4, 0x7f
; GFX12-GI-NEXT: s_min_i32 s1, s1, 0x7f
+; GFX12-GI-NEXT: s_min_i32 s5, s5, 0x7f
; GFX12-GI-NEXT: s_min_i32 s2, s2, 0x7f
+; GFX12-GI-NEXT: s_min_i32 s6, s6, 0x7f
; GFX12-GI-NEXT: s_min_i32 s3, s3, 0x7f
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s7, s7, 0x7f
; GFX12-GI-NEXT: s_max_i32 s0, s0, 0xffffff80
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_max_i32 s4, s4, 0xffffff80
; GFX12-GI-NEXT: s_max_i32 s1, s1, 0xffffff80
+; GFX12-GI-NEXT: s_max_i32 s5, s5, 0xffffff80
; GFX12-GI-NEXT: s_max_i32 s2, s2, 0xffffff80
+; GFX12-GI-NEXT: s_max_i32 s6, s6, 0xffffff80
; GFX12-GI-NEXT: s_max_i32 s3, s3, 0xffffff80
+; GFX12-GI-NEXT: s_max_i32 s7, s7, 0xffffff80
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s0
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s1
-; GFX12-GI-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s2
-; GFX12-GI-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s3
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s5
+; GFX12-GI-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s6
+; GFX12-GI-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i8> @llvm.fptosi.sat.v8f16.v8i8(<8 x half> %f)
ret <8 x i8> %x
@@ -6148,14 +6219,19 @@ define <8 x i16> @test_s_signed_v8f16_v8i16(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s4, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s5, s1, 16
+; GFX12-GI-NEXT: s_lshr_b32 s6, s2, 16
+; GFX12-GI-NEXT: s_lshr_b32 s7, s3, 16
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.h, s4
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.l, s1
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.h, s5
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.l, s2
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.h, s6
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v3.l, s3
-; GFX12-GI-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX12-GI-NEXT: v_mov_b16_e32 v1.h, v1.l
-; GFX12-GI-NEXT: v_mov_b16_e32 v2.h, v2.l
-; GFX12-GI-NEXT: v_mov_b16_e32 v3.h, v3.l
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v3.h, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i16> @llvm.fptosi.sat.v8f16.v8i16(<8 x half> %f)
ret <8 x i16> %x
@@ -6345,24 +6421,37 @@ define <8 x i64> @test_s_signed_v8f16_v8i64(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s4, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s5, s1, 16
+; GFX12-GI-NEXT: s_lshr_b32 s6, s2, 16
+; GFX12-GI-NEXT: s_lshr_b32 s7, s3, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s4, s4
; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s5, s5
; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s6, s6
; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s7, s7
; GFX12-GI-NEXT: s_cvt_i32_f32 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_i32_f32 s4, s4
; GFX12-GI-NEXT: s_cvt_i32_f32 s1, s1
+; GFX12-GI-NEXT: s_cvt_i32_f32 s5, s5
; GFX12-GI-NEXT: s_cvt_i32_f32 s2, s2
+; GFX12-GI-NEXT: s_cvt_i32_f32 s6, s6
; GFX12-GI-NEXT: s_cvt_i32_f32 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_i32_f32 s7, s7
; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, 0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, 0
; GFX12-GI-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, 0
-; GFX12-GI-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v7, 0
+; GFX12-GI-NEXT: v_dual_mov_b32 v6, s5 :: v_dual_mov_b32 v7, 0
; GFX12-GI-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, 0
-; GFX12-GI-NEXT: v_dual_mov_b32 v10, s2 :: v_dual_mov_b32 v11, 0
+; GFX12-GI-NEXT: v_dual_mov_b32 v10, s6 :: v_dual_mov_b32 v11, 0
; GFX12-GI-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v13, 0
-; GFX12-GI-NEXT: v_dual_mov_b32 v14, s3 :: v_dual_mov_b32 v15, 0
+; GFX12-GI-NEXT: v_dual_mov_b32 v14, s7 :: v_dual_mov_b32 v15, 0
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i64> @llvm.fptosi.sat.v8f16.v8i64(<8 x half> %f)
ret <8 x i64> %x
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
index ab7fd327e2c58..f953dc4cd15f5 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
@@ -3119,17 +3119,26 @@ define <4 x i1> @test_s_unsigned_v4f16_v4i1(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
-; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
; GFX12-GI-NEXT: s_cvt_u32_f32 s0, s0
-; GFX12-GI-NEXT: s_cvt_u32_f32 s1, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_u32_f32 s2, s2
+; GFX12-GI-NEXT: s_cvt_u32_f32 s1, s1
+; GFX12-GI-NEXT: s_cvt_u32_f32 s3, s3
; GFX12-GI-NEXT: s_min_u32 s0, s0, 1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_u32 s2, s2, 1
; GFX12-GI-NEXT: s_min_u32 s1, s1, 1
+; GFX12-GI-NEXT: s_min_u32 s3, s3, 1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s0
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s1
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i1> @llvm.fptoui.sat.v4f16.v4i1(<4 x half> %f)
ret <4 x i1> %x
@@ -3333,17 +3342,26 @@ define <4 x i8> @test_s_unsigned_v4f16_v4i8(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
-; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
; GFX12-GI-NEXT: s_cvt_u32_f32 s0, s0
-; GFX12-GI-NEXT: s_cvt_u32_f32 s1, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_u32_f32 s2, s2
+; GFX12-GI-NEXT: s_cvt_u32_f32 s1, s1
+; GFX12-GI-NEXT: s_cvt_u32_f32 s3, s3
; GFX12-GI-NEXT: s_min_u32 s0, s0, 0xff
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_u32 s2, s2, 0xff
; GFX12-GI-NEXT: s_min_u32 s1, s1, 0xff
+; GFX12-GI-NEXT: s_min_u32 s3, s3, 0xff
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s0
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s1
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i8> @llvm.fptoui.sat.v4f16.v4i8(<4 x half> %f)
ret <4 x i8> %x
@@ -3480,10 +3498,13 @@ define <4 x i16> @test_s_unsigned_v4f16_v4i16(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.h, s2
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.l, s1
-; GFX12-GI-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX12-GI-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.h, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i16> @llvm.fptoui.sat.v4f16.v4i16(<4 x half> %f)
ret <4 x i16> %x
@@ -3596,19 +3617,23 @@ define <4 x i64> @test_s_unsigned_v4f16_v4i64(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
-; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
-; GFX12-GI-NEXT: v_mov_b32_e32 v5, 0
-; GFX12-GI-NEXT: v_mov_b32_e32 v7, 0
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
; GFX12-GI-NEXT: s_cvt_u32_f32 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_u32_f32 s2, s2
; GFX12-GI-NEXT: s_cvt_u32_f32 s1, s1
-; GFX12-GI-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-GI-NEXT: s_cvt_u32_f32 s3, s3
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s0
-; GFX12-GI-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GI-NEXT: v_mov_b32_e32 v4, s1
-; GFX12-GI-NEXT: v_mov_b32_e32 v6, s1
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, 0
+; GFX12-GI-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, 0
+; GFX12-GI-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, 0
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i64> @llvm.fptoui.sat.v4f16.v4i64(<4 x half> %f)
ret <4 x i64> %x
@@ -5112,25 +5137,42 @@ define <8 x i1> @test_s_unsigned_v8f16_v8i1(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s4, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s5, s1, 16
+; GFX12-GI-NEXT: s_lshr_b32 s6, s2, 16
+; GFX12-GI-NEXT: s_lshr_b32 s7, s3, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s4, s4
; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s5, s5
; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s6, s6
; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s7, s7
; GFX12-GI-NEXT: s_cvt_u32_f32 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_u32_f32 s4, s4
; GFX12-GI-NEXT: s_cvt_u32_f32 s1, s1
+; GFX12-GI-NEXT: s_cvt_u32_f32 s5, s5
; GFX12-GI-NEXT: s_cvt_u32_f32 s2, s2
+; GFX12-GI-NEXT: s_cvt_u32_f32 s6, s6
; GFX12-GI-NEXT: s_cvt_u32_f32 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_u32_f32 s7, s7
; GFX12-GI-NEXT: s_min_u32 s0, s0, 1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_u32 s4, s4, 1
; GFX12-GI-NEXT: s_min_u32 s1, s1, 1
+; GFX12-GI-NEXT: s_min_u32 s5, s5, 1
; GFX12-GI-NEXT: s_min_u32 s2, s2, 1
+; GFX12-GI-NEXT: s_min_u32 s6, s6, 1
; GFX12-GI-NEXT: s_min_u32 s3, s3, 1
+; GFX12-GI-NEXT: s_min_u32 s7, s7, 1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s0
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s1
-; GFX12-GI-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s2
-; GFX12-GI-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s3
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s5
+; GFX12-GI-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s6
+; GFX12-GI-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i1> @llvm.fptoui.sat.v8f16.v8i1(<8 x half> %f)
ret <8 x i1> %x
@@ -5488,25 +5530,42 @@ define <8 x i8> @test_s_unsigned_v8f16_v8i8(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s4, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s5, s1, 16
+; GFX12-GI-NEXT: s_lshr_b32 s6, s2, 16
+; GFX12-GI-NEXT: s_lshr_b32 s7, s3, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s4, s4
; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s5, s5
; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s6, s6
; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s7, s7
; GFX12-GI-NEXT: s_cvt_u32_f32 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_u32_f32 s4, s4
; GFX12-GI-NEXT: s_cvt_u32_f32 s1, s1
+; GFX12-GI-NEXT: s_cvt_u32_f32 s5, s5
; GFX12-GI-NEXT: s_cvt_u32_f32 s2, s2
+; GFX12-GI-NEXT: s_cvt_u32_f32 s6, s6
; GFX12-GI-NEXT: s_cvt_u32_f32 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_u32_f32 s7, s7
; GFX12-GI-NEXT: s_min_u32 s0, s0, 0xff
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_u32 s4, s4, 0xff
; GFX12-GI-NEXT: s_min_u32 s1, s1, 0xff
+; GFX12-GI-NEXT: s_min_u32 s5, s5, 0xff
; GFX12-GI-NEXT: s_min_u32 s2, s2, 0xff
+; GFX12-GI-NEXT: s_min_u32 s6, s6, 0xff
; GFX12-GI-NEXT: s_min_u32 s3, s3, 0xff
+; GFX12-GI-NEXT: s_min_u32 s7, s7, 0xff
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s0
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s1
-; GFX12-GI-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s2
-; GFX12-GI-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s3
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s5
+; GFX12-GI-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s6
+; GFX12-GI-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i8> @llvm.fptoui.sat.v8f16.v8i8(<8 x half> %f)
ret <8 x i8> %x
@@ -5724,14 +5783,19 @@ define <8 x i16> @test_s_unsigned_v8f16_v8i16(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s4, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s5, s1, 16
+; GFX12-GI-NEXT: s_lshr_b32 s6, s2, 16
+; GFX12-GI-NEXT: s_lshr_b32 s7, s3, 16
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.h, s4
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.l, s1
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.h, s5
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, s2
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.h, s6
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v3.l, s3
-; GFX12-GI-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX12-GI-NEXT: v_mov_b16_e32 v1.h, v1.l
-; GFX12-GI-NEXT: v_mov_b16_e32 v2.h, v2.l
-; GFX12-GI-NEXT: v_mov_b16_e32 v3.h, v3.l
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v3.h, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i16> @llvm.fptoui.sat.v8f16.v8i16(<8 x half> %f)
ret <8 x i16> %x
@@ -5912,24 +5976,37 @@ define <8 x i64> @test_s_unsigned_v8f16_v8i64(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s4, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s5, s1, 16
+; GFX12-GI-NEXT: s_lshr_b32 s6, s2, 16
+; GFX12-GI-NEXT: s_lshr_b32 s7, s3, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s4, s4
; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s5, s5
; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s6, s6
; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s7, s7
; GFX12-GI-NEXT: s_cvt_u32_f32 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_u32_f32 s4, s4
; GFX12-GI-NEXT: s_cvt_u32_f32 s1, s1
+; GFX12-GI-NEXT: s_cvt_u32_f32 s5, s5
; GFX12-GI-NEXT: s_cvt_u32_f32 s2, s2
+; GFX12-GI-NEXT: s_cvt_u32_f32 s6, s6
; GFX12-GI-NEXT: s_cvt_u32_f32 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_u32_f32 s7, s7
; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, 0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, 0
; GFX12-GI-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, 0
-; GFX12-GI-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v7, 0
+; GFX12-GI-NEXT: v_dual_mov_b32 v6, s5 :: v_dual_mov_b32 v7, 0
; GFX12-GI-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, 0
-; GFX12-GI-NEXT: v_dual_mov_b32 v10, s2 :: v_dual_mov_b32 v11, 0
+; GFX12-GI-NEXT: v_dual_mov_b32 v10, s6 :: v_dual_mov_b32 v11, 0
; GFX12-GI-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v13, 0
-; GFX12-GI-NEXT: v_dual_mov_b32 v14, s3 :: v_dual_mov_b32 v15, 0
+; GFX12-GI-NEXT: v_dual_mov_b32 v14, s7 :: v_dual_mov_b32 v15, 0
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i64> @llvm.fptoui.sat.v8f16.v8i64(<8 x half> %f)
ret <8 x i64> %x
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
index 614566a230f68..be3bdd7d9cd9e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
@@ -117,11 +117,13 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16(i32 inreg %node_ptr, f
; GFX12-GISEL-TRUE16-LABEL: image_bvh_intersect_ray_a16:
; GFX12-GISEL-TRUE16: ; %bb.0: ; %main_body
; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s20, s2
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s21, s3
+; GFX12-GISEL-TRUE16-NEXT: s_lshr_b32 s2, s5, 16
+; GFX12-GISEL-TRUE16-NEXT: s_lshr_b32 s3, s7, 16
; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s22, s4
; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s4, s7, s5
-; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s21, s3
+; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s3, s2
; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s6, s8, s6
-; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-GISEL-TRUE16-NEXT: v_dual_mov_b32 v0, s20 :: v_dual_mov_b32 v1, s21
; GFX12-GISEL-TRUE16-NEXT: v_dual_mov_b32 v2, s22 :: v_dual_mov_b32 v3, s4
; GFX12-GISEL-TRUE16-NEXT: v_dual_mov_b32 v6, s0 :: v_dual_mov_b32 v7, s1
@@ -258,12 +260,14 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16(i64 inreg %node_ptr,
;
; GFX12-GISEL-TRUE16-LABEL: image_bvh64_intersect_ray_a16:
; GFX12-GISEL-TRUE16: ; %bb.0: ; %main_body
-; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s21, s4
-; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s4, s8, s6
; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s20, s3
; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s22, s5
+; GFX12-GISEL-TRUE16-NEXT: s_lshr_b32 s3, s6, 16
+; GFX12-GISEL-TRUE16-NEXT: s_lshr_b32 s5, s8, 16
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s21, s4
+; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s4, s8, s6
+; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s5, s3
; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s6, s9, s7
-; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-GISEL-TRUE16-NEXT: v_dual_mov_b32 v0, s20 :: v_dual_mov_b32 v3, s4
; GFX12-GISEL-TRUE16-NEXT: v_dual_mov_b32 v7, s1 :: v_dual_mov_b32 v6, s0
; GFX12-GISEL-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s21
diff --git a/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir b/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir
index 176d7752133cf..ac25b6a760bf0 100644
--- a/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir
+++ b/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir
@@ -210,21 +210,6 @@ body: |
...
----
-name: fold_sreg_64_hi16_to_sgpr_lo16
-body: |
- bb.0:
-
- ; GCN-LABEL: name: fold_sreg_64_hi16_to_sgpr_lo16
- ; GCN: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 1125912791875585
- ; GCN-NEXT: $sgpr0 = S_MOV_B32 2
- ; GCN-NEXT: SI_RETURN_TO_EPILOG $sgpr0_lo16
- %0:sreg_64 = S_MOV_B64_IMM_PSEUDO 1125912791875585
- $sgpr0_lo16 = COPY killed %0.hi16
- SI_RETURN_TO_EPILOG $sgpr0_lo16
-
-...
-
---
name: fold_sreg_64_sub1_lo16_to_sgpr_lo16
body: |
@@ -820,20 +805,6 @@ body: |
%1:sgpr_lo16 = COPY killed %0.lo16
SI_RETURN_TO_EPILOG %1
-...
----
-name: fold_simm_16_sub_to_hi_from_mov_64_inline_imm_virt_sgpr16
-body: |
- bb.0:
-
- ; GCN-LABEL: name: fold_simm_16_sub_to_hi_from_mov_64_inline_imm_virt_sgpr16
- ; GCN: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 64
- ; GCN-NEXT: [[COPY:%[0-9]+]]:sgpr_lo16 = COPY killed [[S_MOV_B64_]].hi16
- ; GCN-NEXT: SI_RETURN_TO_EPILOG [[COPY]]
- %0:sreg_64 = S_MOV_B64 64
- %1:sgpr_lo16 = COPY killed %0.hi16
- SI_RETURN_TO_EPILOG %1
-
...
---
@@ -850,17 +821,3 @@ body: |
SI_RETURN_TO_EPILOG $sgpr0_lo16
...
----
-name: fold_simm_16_sub_to_hi_from_mov_64_inline_imm_phys_sgpr16_lo
-body: |
- bb.0:
-
- ; GCN-LABEL: name: fold_simm_16_sub_to_hi_from_mov_64_inline_imm_phys_sgpr16_lo
- ; GCN: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 64
- ; GCN-NEXT: $sgpr0 = S_MOV_B32 0
- ; GCN-NEXT: SI_RETURN_TO_EPILOG $sgpr0_lo16
- %0:sreg_64 = S_MOV_B64 64
- $sgpr0_lo16 = COPY killed %0.hi16
- SI_RETURN_TO_EPILOG $sgpr0_lo16
-
-...
>From 0aeeb9279c5ba0385e9cbfbc94b93219b7a59b70 Mon Sep 17 00:00:00 2001
From: Igor Wodiany <igor.wodiany at amd.com>
Date: Thu, 26 Mar 2026 16:56:34 +0000
Subject: [PATCH 2/2] Replace reg fix
---
llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
index 4b629534caa6c..23b8145c79464 100644
--- a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
@@ -245,7 +245,7 @@ bool InstructionSelect::selectMachineFunction(MachineFunction &MF) {
Register DstReg = MI.getOperand(0).getReg();
unsigned SrcSubIdx = MI.getOperand(1).getSubReg();
unsigned DstSubIdx = MI.getOperand(0).getSubReg();
- if (!SrcReg.isVirtual() || !DstReg.isVirtual() || SrcSubIdx != DstSubIdx)
+ if (!SrcReg.isVirtual() || !DstReg.isVirtual() || SrcSubIdx || DstSubIdx)
continue;
const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);
More information about the llvm-commits
mailing list