[llvm] [AMDGPU][GlobalISel] Guard against use of disallowed sub-registers (PR #188781)
Igor Wodiany via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 13 03:23:22 PDT 2026
https://github.com/IgWod updated https://github.com/llvm/llvm-project/pull/188781
>From 3053992ae6bce67990a846b9ba8780d5562490c4 Mon Sep 17 00:00:00 2001
From: Igor Wodiany <igor.wodiany at amd.com>
Date: Thu, 26 Mar 2026 14:57:05 +0000
Subject: [PATCH 1/9] [AMDGPU][GlobalISel] Guard aginst use of `hi16` for SGPR
registers
This patch makes several interconnected changes that need to be
merged together:
1) It updates redundant copies elimination after isel so it accounts
for sub-registers. This prevents `lo16` and `hi16` being treated as
the same values.
2) It updates machine verification to prevent use of `hi16` with SGPR
registers as it is not supported in pipelines following the isel.
3) Update `selectG_UNMERGE_VALUES` so instead of generating `hi16` for
SGPR it shifts higher bits into the destination register.
Tests:
1) `fto[s/u]i-sat-vector.ll`: The correct number of conversions is now generated
as higher 16-bits are handled correctly; however, it introduces
`lshr` instructions. This should be resolved in 1A8A8287 by ennabling
`s_cvt_hi_`.
2) `llvm.amdgcn.intersect_ray.ll`: GlobalISel generates a correct number
of pack instructions now; however the codegen is sub-optimal as it
should replace `lshr` instructions with `s_pack_hh_`. (future work)
3) `peephole-fold-imm.mir`: This test was explicitly using `hi16` so I
removed offending tests.
---
.../CodeGen/GlobalISel/InstructionSelect.cpp | 17 +-
.../AMDGPU/AMDGPUInstructionSelector.cpp | 10 +-
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 29 ++-
llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll | 177 +++++++++++++-----
llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll | 157 ++++++++++++----
.../AMDGPU/llvm.amdgcn.intersect_ray.ll | 14 +-
.../test/CodeGen/AMDGPU/peephole-fold-imm.mir | 43 -----
7 files changed, 304 insertions(+), 143 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
index 5fb31a04e5fd0..4b629534caa6c 100644
--- a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
@@ -243,13 +243,16 @@ bool InstructionSelect::selectMachineFunction(MachineFunction &MF) {
continue;
Register SrcReg = MI.getOperand(1).getReg();
Register DstReg = MI.getOperand(0).getReg();
- if (SrcReg.isVirtual() && DstReg.isVirtual()) {
- auto SrcRC = MRI.getRegClass(SrcReg);
- auto DstRC = MRI.getRegClass(DstReg);
- if (SrcRC == DstRC) {
- MRI.replaceRegWith(DstReg, SrcReg);
- MI.eraseFromParent();
- }
+ unsigned SrcSubIdx = MI.getOperand(1).getSubReg();
+ unsigned DstSubIdx = MI.getOperand(0).getSubReg();
+ if (!SrcReg.isVirtual() || !DstReg.isVirtual() || SrcSubIdx != DstSubIdx)
+ continue;
+
+ const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);
+ const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
+ if (SrcRC == DstRC) {
+ MRI.replaceRegWith(DstReg, SrcReg);
+ MI.eraseFromParent();
}
}
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index f5747488225c5..cd0a78660dac0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -730,8 +730,14 @@ bool AMDGPUInstructionSelector::selectG_UNMERGE_VALUES(MachineInstr &MI) const {
ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(SrcRC, DstSize / 8);
for (int I = 0, E = NumDst; I != E; ++I) {
MachineOperand &Dst = MI.getOperand(I);
- BuildMI(*BB, &MI, DL, TII.get(TargetOpcode::COPY), Dst.getReg())
- .addReg(SrcReg, {}, SubRegs[I]);
+ // hi16:sreg_32 is not allowed so explicitly shift upper 16-bits.
+ if (SrcBank->getID() == AMDGPU::SGPRRegBankID && SubRegs[I] == AMDGPU::hi16)
+ BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_LSHR_B32), Dst.getReg())
+ .addReg(SrcReg)
+ .addImm(16);
+ else
+ BuildMI(*BB, &MI, DL, TII.get(TargetOpcode::COPY), Dst.getReg())
+ .addReg(SrcReg, {}, SubRegs[I]);
// Make sure the subregister index is valid for the source register.
SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubRegs[I]);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index df2700d414893..254dab030f25b 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5163,6 +5163,19 @@ static bool isSubRegOf(const SIRegisterInfo &TRI,
SubReg.getReg() == SuperVec.getReg();
}
+/// Check if given MachineOperand is an SGPR with SubReg hi16.
+static bool isSGPRHi16(const MachineRegisterInfo &MRI, const SIRegisterInfo &RI,
+ const MachineOperand &MO) {
+ if (!MO.isReg())
+ return false;
+
+ Register Reg = MO.getReg();
+ if (!Reg || !Reg.isVirtual() || !MRI.getRegClassOrNull(Reg))
+ return false;
+
+ return RI.isSGPRReg(MRI, Reg) && MO.getSubReg() == AMDGPU::hi16;
+}
+
// Verify the illegal copy from vector register to SGPR for generic opcode COPY
bool SIInstrInfo::verifyCopy(const MachineInstr &MI,
const MachineRegisterInfo &MRI,
@@ -5192,7 +5205,20 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
if (!MRI.isSSA() && MI.isCopy())
return verifyCopy(MI, MRI, ErrInfo);
- if (SIInstrInfo::isGenericOpcode(Opcode))
+ if (SIInstrInfo::isGenericOpcode(Opcode) && !MI.isCopy())
+ return true;
+
+ // Verify that none of the operands in either AMDGPU instruction or COPY
+ // is SGPR hi16.
+ const MCInstrDesc &Desc = get(Opcode);
+ for (int I = 0, E = Desc.getNumOperands(); I != E; ++I) {
+ if (isSGPRHi16(MRI, RI, MI.getOperand(I))) {
+ ErrInfo = "SGPR cannot use hi16 subreg";
+ return false;
+ }
+ }
+
+ if (MI.isCopy())
return true;
int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
@@ -5208,7 +5234,6 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
}
// Make sure the number of operands is correct.
- const MCInstrDesc &Desc = get(Opcode);
if (!Desc.isVariadic() &&
Desc.getNumOperands() != MI.getNumExplicitOperands()) {
ErrInfo = "Instruction has wrong number of operands.";
diff --git a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
index 590c2c748bfdc..6063bafe48891 100644
--- a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
@@ -3348,20 +3348,31 @@ define <4 x i1> @test_s_signed_v4f16_v4i1(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
-; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
; GFX12-GI-NEXT: s_cvt_i32_f32 s0, s0
-; GFX12-GI-NEXT: s_cvt_i32_f32 s1, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_i32_f32 s2, s2
+; GFX12-GI-NEXT: s_cvt_i32_f32 s1, s1
+; GFX12-GI-NEXT: s_cvt_i32_f32 s3, s3
; GFX12-GI-NEXT: s_min_i32 s0, s0, 0
-; GFX12-GI-NEXT: s_min_i32 s1, s1, 0
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s2, s2, 0
+; GFX12-GI-NEXT: s_min_i32 s1, s1, 0
+; GFX12-GI-NEXT: s_min_i32 s3, s3, 0
; GFX12-GI-NEXT: s_max_i32 s0, s0, -1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_max_i32 s2, s2, -1
; GFX12-GI-NEXT: s_max_i32 s1, s1, -1
+; GFX12-GI-NEXT: s_max_i32 s3, s3, -1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s0
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s1
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i1> @llvm.fptosi.sat.v4f16.v4i1(<4 x half> %f)
ret <4 x i1> %x
@@ -3597,20 +3608,31 @@ define <4 x i8> @test_s_signed_v4f16_v4i8(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
-; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
; GFX12-GI-NEXT: s_cvt_i32_f32 s0, s0
-; GFX12-GI-NEXT: s_cvt_i32_f32 s1, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_i32_f32 s2, s2
+; GFX12-GI-NEXT: s_cvt_i32_f32 s1, s1
+; GFX12-GI-NEXT: s_cvt_i32_f32 s3, s3
; GFX12-GI-NEXT: s_min_i32 s0, s0, 0x7f
-; GFX12-GI-NEXT: s_min_i32 s1, s1, 0x7f
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s2, s2, 0x7f
+; GFX12-GI-NEXT: s_min_i32 s1, s1, 0x7f
+; GFX12-GI-NEXT: s_min_i32 s3, s3, 0x7f
; GFX12-GI-NEXT: s_max_i32 s0, s0, 0xffffff80
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_max_i32 s2, s2, 0xffffff80
; GFX12-GI-NEXT: s_max_i32 s1, s1, 0xffffff80
+; GFX12-GI-NEXT: s_max_i32 s3, s3, 0xffffff80
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s0
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s1
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i8> @llvm.fptosi.sat.v4f16.v4i8(<4 x half> %f)
ret <4 x i8> %x
@@ -3757,10 +3779,13 @@ define <4 x i16> @test_s_signed_v4f16_v4i16(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.h, s2
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.l, s1
-; GFX12-GI-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX12-GI-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.h, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i16> @llvm.fptosi.sat.v4f16.v4i16(<4 x half> %f)
ret <4 x i16> %x
@@ -3878,19 +3903,23 @@ define <4 x i64> @test_s_signed_v4f16_v4i64(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
-; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
-; GFX12-GI-NEXT: v_mov_b32_e32 v5, 0
-; GFX12-GI-NEXT: v_mov_b32_e32 v7, 0
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
; GFX12-GI-NEXT: s_cvt_i32_f32 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_i32_f32 s2, s2
; GFX12-GI-NEXT: s_cvt_i32_f32 s1, s1
-; GFX12-GI-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-GI-NEXT: s_cvt_i32_f32 s3, s3
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s0
-; GFX12-GI-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GI-NEXT: v_mov_b32_e32 v4, s1
-; GFX12-GI-NEXT: v_mov_b32_e32 v6, s1
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, 0
+; GFX12-GI-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, 0
+; GFX12-GI-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, 0
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i64> @llvm.fptosi.sat.v4f16.v4i64(<4 x half> %f)
ret <4 x i64> %x
@@ -5463,30 +5492,51 @@ define <8 x i1> @test_s_signed_v8f16_v8i1(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s4, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s5, s1, 16
+; GFX12-GI-NEXT: s_lshr_b32 s6, s2, 16
+; GFX12-GI-NEXT: s_lshr_b32 s7, s3, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s4, s4
; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s5, s5
; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s6, s6
; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s7, s7
; GFX12-GI-NEXT: s_cvt_i32_f32 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_i32_f32 s4, s4
; GFX12-GI-NEXT: s_cvt_i32_f32 s1, s1
+; GFX12-GI-NEXT: s_cvt_i32_f32 s5, s5
; GFX12-GI-NEXT: s_cvt_i32_f32 s2, s2
+; GFX12-GI-NEXT: s_cvt_i32_f32 s6, s6
; GFX12-GI-NEXT: s_cvt_i32_f32 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_i32_f32 s7, s7
; GFX12-GI-NEXT: s_min_i32 s0, s0, 0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s4, s4, 0
; GFX12-GI-NEXT: s_min_i32 s1, s1, 0
+; GFX12-GI-NEXT: s_min_i32 s5, s5, 0
; GFX12-GI-NEXT: s_min_i32 s2, s2, 0
+; GFX12-GI-NEXT: s_min_i32 s6, s6, 0
; GFX12-GI-NEXT: s_min_i32 s3, s3, 0
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s7, s7, 0
; GFX12-GI-NEXT: s_max_i32 s0, s0, -1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_max_i32 s4, s4, -1
; GFX12-GI-NEXT: s_max_i32 s1, s1, -1
+; GFX12-GI-NEXT: s_max_i32 s5, s5, -1
; GFX12-GI-NEXT: s_max_i32 s2, s2, -1
+; GFX12-GI-NEXT: s_max_i32 s6, s6, -1
; GFX12-GI-NEXT: s_max_i32 s3, s3, -1
+; GFX12-GI-NEXT: s_max_i32 s7, s7, -1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s0
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s1
-; GFX12-GI-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s2
-; GFX12-GI-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s3
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s5
+; GFX12-GI-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s6
+; GFX12-GI-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i1> @llvm.fptosi.sat.v8f16.v8i1(<8 x half> %f)
ret <8 x i1> %x
@@ -5891,30 +5941,51 @@ define <8 x i8> @test_s_signed_v8f16_v8i8(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s4, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s5, s1, 16
+; GFX12-GI-NEXT: s_lshr_b32 s6, s2, 16
+; GFX12-GI-NEXT: s_lshr_b32 s7, s3, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s4, s4
; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s5, s5
; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s6, s6
; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s7, s7
; GFX12-GI-NEXT: s_cvt_i32_f32 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_i32_f32 s4, s4
; GFX12-GI-NEXT: s_cvt_i32_f32 s1, s1
+; GFX12-GI-NEXT: s_cvt_i32_f32 s5, s5
; GFX12-GI-NEXT: s_cvt_i32_f32 s2, s2
+; GFX12-GI-NEXT: s_cvt_i32_f32 s6, s6
; GFX12-GI-NEXT: s_cvt_i32_f32 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_i32_f32 s7, s7
; GFX12-GI-NEXT: s_min_i32 s0, s0, 0x7f
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s4, s4, 0x7f
; GFX12-GI-NEXT: s_min_i32 s1, s1, 0x7f
+; GFX12-GI-NEXT: s_min_i32 s5, s5, 0x7f
; GFX12-GI-NEXT: s_min_i32 s2, s2, 0x7f
+; GFX12-GI-NEXT: s_min_i32 s6, s6, 0x7f
; GFX12-GI-NEXT: s_min_i32 s3, s3, 0x7f
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_i32 s7, s7, 0x7f
; GFX12-GI-NEXT: s_max_i32 s0, s0, 0xffffff80
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_max_i32 s4, s4, 0xffffff80
; GFX12-GI-NEXT: s_max_i32 s1, s1, 0xffffff80
+; GFX12-GI-NEXT: s_max_i32 s5, s5, 0xffffff80
; GFX12-GI-NEXT: s_max_i32 s2, s2, 0xffffff80
+; GFX12-GI-NEXT: s_max_i32 s6, s6, 0xffffff80
; GFX12-GI-NEXT: s_max_i32 s3, s3, 0xffffff80
+; GFX12-GI-NEXT: s_max_i32 s7, s7, 0xffffff80
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s0
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s1
-; GFX12-GI-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s2
-; GFX12-GI-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s3
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s5
+; GFX12-GI-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s6
+; GFX12-GI-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i8> @llvm.fptosi.sat.v8f16.v8i8(<8 x half> %f)
ret <8 x i8> %x
@@ -6148,14 +6219,19 @@ define <8 x i16> @test_s_signed_v8f16_v8i16(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s4, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s5, s1, 16
+; GFX12-GI-NEXT: s_lshr_b32 s6, s2, 16
+; GFX12-GI-NEXT: s_lshr_b32 s7, s3, 16
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.h, s4
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.l, s1
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.h, s5
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.l, s2
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.h, s6
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v3.l, s3
-; GFX12-GI-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX12-GI-NEXT: v_mov_b16_e32 v1.h, v1.l
-; GFX12-GI-NEXT: v_mov_b16_e32 v2.h, v2.l
-; GFX12-GI-NEXT: v_mov_b16_e32 v3.h, v3.l
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v3.h, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i16> @llvm.fptosi.sat.v8f16.v8i16(<8 x half> %f)
ret <8 x i16> %x
@@ -6345,24 +6421,37 @@ define <8 x i64> @test_s_signed_v8f16_v8i64(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s4, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s5, s1, 16
+; GFX12-GI-NEXT: s_lshr_b32 s6, s2, 16
+; GFX12-GI-NEXT: s_lshr_b32 s7, s3, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s4, s4
; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s5, s5
; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s6, s6
; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s7, s7
; GFX12-GI-NEXT: s_cvt_i32_f32 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_i32_f32 s4, s4
; GFX12-GI-NEXT: s_cvt_i32_f32 s1, s1
+; GFX12-GI-NEXT: s_cvt_i32_f32 s5, s5
; GFX12-GI-NEXT: s_cvt_i32_f32 s2, s2
+; GFX12-GI-NEXT: s_cvt_i32_f32 s6, s6
; GFX12-GI-NEXT: s_cvt_i32_f32 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_i32_f32 s7, s7
; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, 0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, 0
; GFX12-GI-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, 0
-; GFX12-GI-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v7, 0
+; GFX12-GI-NEXT: v_dual_mov_b32 v6, s5 :: v_dual_mov_b32 v7, 0
; GFX12-GI-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, 0
-; GFX12-GI-NEXT: v_dual_mov_b32 v10, s2 :: v_dual_mov_b32 v11, 0
+; GFX12-GI-NEXT: v_dual_mov_b32 v10, s6 :: v_dual_mov_b32 v11, 0
; GFX12-GI-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v13, 0
-; GFX12-GI-NEXT: v_dual_mov_b32 v14, s3 :: v_dual_mov_b32 v15, 0
+; GFX12-GI-NEXT: v_dual_mov_b32 v14, s7 :: v_dual_mov_b32 v15, 0
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i64> @llvm.fptosi.sat.v8f16.v8i64(<8 x half> %f)
ret <8 x i64> %x
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
index ab7fd327e2c58..f953dc4cd15f5 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
@@ -3119,17 +3119,26 @@ define <4 x i1> @test_s_unsigned_v4f16_v4i1(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
-; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
; GFX12-GI-NEXT: s_cvt_u32_f32 s0, s0
-; GFX12-GI-NEXT: s_cvt_u32_f32 s1, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_u32_f32 s2, s2
+; GFX12-GI-NEXT: s_cvt_u32_f32 s1, s1
+; GFX12-GI-NEXT: s_cvt_u32_f32 s3, s3
; GFX12-GI-NEXT: s_min_u32 s0, s0, 1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_u32 s2, s2, 1
; GFX12-GI-NEXT: s_min_u32 s1, s1, 1
+; GFX12-GI-NEXT: s_min_u32 s3, s3, 1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s0
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s1
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i1> @llvm.fptoui.sat.v4f16.v4i1(<4 x half> %f)
ret <4 x i1> %x
@@ -3333,17 +3342,26 @@ define <4 x i8> @test_s_unsigned_v4f16_v4i8(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
-; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
; GFX12-GI-NEXT: s_cvt_u32_f32 s0, s0
-; GFX12-GI-NEXT: s_cvt_u32_f32 s1, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_u32_f32 s2, s2
+; GFX12-GI-NEXT: s_cvt_u32_f32 s1, s1
+; GFX12-GI-NEXT: s_cvt_u32_f32 s3, s3
; GFX12-GI-NEXT: s_min_u32 s0, s0, 0xff
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_u32 s2, s2, 0xff
; GFX12-GI-NEXT: s_min_u32 s1, s1, 0xff
+; GFX12-GI-NEXT: s_min_u32 s3, s3, 0xff
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s0
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s1
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i8> @llvm.fptoui.sat.v4f16.v4i8(<4 x half> %f)
ret <4 x i8> %x
@@ -3480,10 +3498,13 @@ define <4 x i16> @test_s_unsigned_v4f16_v4i16(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.h, s2
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.l, s1
-; GFX12-GI-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX12-GI-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.h, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i16> @llvm.fptoui.sat.v4f16.v4i16(<4 x half> %f)
ret <4 x i16> %x
@@ -3596,19 +3617,23 @@ define <4 x i64> @test_s_unsigned_v4f16_v4i64(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s2, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s3, s1, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
-; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
-; GFX12-GI-NEXT: v_mov_b32_e32 v5, 0
-; GFX12-GI-NEXT: v_mov_b32_e32 v7, 0
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
; GFX12-GI-NEXT: s_cvt_u32_f32 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_u32_f32 s2, s2
; GFX12-GI-NEXT: s_cvt_u32_f32 s1, s1
-; GFX12-GI-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-GI-NEXT: s_cvt_u32_f32 s3, s3
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s0
-; GFX12-GI-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GI-NEXT: v_mov_b32_e32 v4, s1
-; GFX12-GI-NEXT: v_mov_b32_e32 v6, s1
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, 0
+; GFX12-GI-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, 0
+; GFX12-GI-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, 0
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i64> @llvm.fptoui.sat.v4f16.v4i64(<4 x half> %f)
ret <4 x i64> %x
@@ -5112,25 +5137,42 @@ define <8 x i1> @test_s_unsigned_v8f16_v8i1(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s4, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s5, s1, 16
+; GFX12-GI-NEXT: s_lshr_b32 s6, s2, 16
+; GFX12-GI-NEXT: s_lshr_b32 s7, s3, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s4, s4
; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s5, s5
; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s6, s6
; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s7, s7
; GFX12-GI-NEXT: s_cvt_u32_f32 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_u32_f32 s4, s4
; GFX12-GI-NEXT: s_cvt_u32_f32 s1, s1
+; GFX12-GI-NEXT: s_cvt_u32_f32 s5, s5
; GFX12-GI-NEXT: s_cvt_u32_f32 s2, s2
+; GFX12-GI-NEXT: s_cvt_u32_f32 s6, s6
; GFX12-GI-NEXT: s_cvt_u32_f32 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_u32_f32 s7, s7
; GFX12-GI-NEXT: s_min_u32 s0, s0, 1
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_u32 s4, s4, 1
; GFX12-GI-NEXT: s_min_u32 s1, s1, 1
+; GFX12-GI-NEXT: s_min_u32 s5, s5, 1
; GFX12-GI-NEXT: s_min_u32 s2, s2, 1
+; GFX12-GI-NEXT: s_min_u32 s6, s6, 1
; GFX12-GI-NEXT: s_min_u32 s3, s3, 1
+; GFX12-GI-NEXT: s_min_u32 s7, s7, 1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s0
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s1
-; GFX12-GI-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s2
-; GFX12-GI-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s3
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s5
+; GFX12-GI-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s6
+; GFX12-GI-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i1> @llvm.fptoui.sat.v8f16.v8i1(<8 x half> %f)
ret <8 x i1> %x
@@ -5488,25 +5530,42 @@ define <8 x i8> @test_s_unsigned_v8f16_v8i8(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s4, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s5, s1, 16
+; GFX12-GI-NEXT: s_lshr_b32 s6, s2, 16
+; GFX12-GI-NEXT: s_lshr_b32 s7, s3, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s4, s4
; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s5, s5
; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s6, s6
; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s7, s7
; GFX12-GI-NEXT: s_cvt_u32_f32 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_u32_f32 s4, s4
; GFX12-GI-NEXT: s_cvt_u32_f32 s1, s1
+; GFX12-GI-NEXT: s_cvt_u32_f32 s5, s5
; GFX12-GI-NEXT: s_cvt_u32_f32 s2, s2
+; GFX12-GI-NEXT: s_cvt_u32_f32 s6, s6
; GFX12-GI-NEXT: s_cvt_u32_f32 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_u32_f32 s7, s7
; GFX12-GI-NEXT: s_min_u32 s0, s0, 0xff
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_min_u32 s4, s4, 0xff
; GFX12-GI-NEXT: s_min_u32 s1, s1, 0xff
+; GFX12-GI-NEXT: s_min_u32 s5, s5, 0xff
; GFX12-GI-NEXT: s_min_u32 s2, s2, 0xff
+; GFX12-GI-NEXT: s_min_u32 s6, s6, 0xff
; GFX12-GI-NEXT: s_min_u32 s3, s3, 0xff
+; GFX12-GI-NEXT: s_min_u32 s7, s7, 0xff
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s0
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s1
-; GFX12-GI-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s2
-; GFX12-GI-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s3
+; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s5
+; GFX12-GI-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s6
+; GFX12-GI-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i8> @llvm.fptoui.sat.v8f16.v8i8(<8 x half> %f)
ret <8 x i8> %x
@@ -5724,14 +5783,19 @@ define <8 x i16> @test_s_unsigned_v8f16_v8i16(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s4, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s5, s1, 16
+; GFX12-GI-NEXT: s_lshr_b32 s6, s2, 16
+; GFX12-GI-NEXT: s_lshr_b32 s7, s3, 16
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.h, s4
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.l, s1
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.h, s5
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, s2
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.h, s6
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v3.l, s3
-; GFX12-GI-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX12-GI-NEXT: v_mov_b16_e32 v1.h, v1.l
-; GFX12-GI-NEXT: v_mov_b16_e32 v2.h, v2.l
-; GFX12-GI-NEXT: v_mov_b16_e32 v3.h, v3.l
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v3.h, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i16> @llvm.fptoui.sat.v8f16.v8i16(<8 x half> %f)
ret <8 x i16> %x
@@ -5912,24 +5976,37 @@ define <8 x i64> @test_s_unsigned_v8f16_v8i64(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: s_lshr_b32 s4, s0, 16
+; GFX12-GI-NEXT: s_lshr_b32 s5, s1, 16
+; GFX12-GI-NEXT: s_lshr_b32 s6, s2, 16
+; GFX12-GI-NEXT: s_lshr_b32 s7, s3, 16
; GFX12-GI-NEXT: s_cvt_f32_f16 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s4, s4
; GFX12-GI-NEXT: s_cvt_f32_f16 s1, s1
+; GFX12-GI-NEXT: s_cvt_f32_f16 s5, s5
; GFX12-GI-NEXT: s_cvt_f32_f16 s2, s2
+; GFX12-GI-NEXT: s_cvt_f32_f16 s6, s6
; GFX12-GI-NEXT: s_cvt_f32_f16 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_f32_f16 s7, s7
; GFX12-GI-NEXT: s_cvt_u32_f32 s0, s0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_u32_f32 s4, s4
; GFX12-GI-NEXT: s_cvt_u32_f32 s1, s1
+; GFX12-GI-NEXT: s_cvt_u32_f32 s5, s5
; GFX12-GI-NEXT: s_cvt_u32_f32 s2, s2
+; GFX12-GI-NEXT: s_cvt_u32_f32 s6, s6
; GFX12-GI-NEXT: s_cvt_u32_f32 s3, s3
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: s_cvt_u32_f32 s7, s7
; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, 0
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, 0
+; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GI-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, 0
; GFX12-GI-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, 0
-; GFX12-GI-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v7, 0
+; GFX12-GI-NEXT: v_dual_mov_b32 v6, s5 :: v_dual_mov_b32 v7, 0
; GFX12-GI-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, 0
-; GFX12-GI-NEXT: v_dual_mov_b32 v10, s2 :: v_dual_mov_b32 v11, 0
+; GFX12-GI-NEXT: v_dual_mov_b32 v10, s6 :: v_dual_mov_b32 v11, 0
; GFX12-GI-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v13, 0
-; GFX12-GI-NEXT: v_dual_mov_b32 v14, s3 :: v_dual_mov_b32 v15, 0
+; GFX12-GI-NEXT: v_dual_mov_b32 v14, s7 :: v_dual_mov_b32 v15, 0
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i64> @llvm.fptoui.sat.v8f16.v8i64(<8 x half> %f)
ret <8 x i64> %x
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
index 614566a230f68..be3bdd7d9cd9e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
@@ -117,11 +117,13 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16(i32 inreg %node_ptr, f
; GFX12-GISEL-TRUE16-LABEL: image_bvh_intersect_ray_a16:
; GFX12-GISEL-TRUE16: ; %bb.0: ; %main_body
; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s20, s2
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s21, s3
+; GFX12-GISEL-TRUE16-NEXT: s_lshr_b32 s2, s5, 16
+; GFX12-GISEL-TRUE16-NEXT: s_lshr_b32 s3, s7, 16
; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s22, s4
; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s4, s7, s5
-; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s21, s3
+; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s3, s2
; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s6, s8, s6
-; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-GISEL-TRUE16-NEXT: v_dual_mov_b32 v0, s20 :: v_dual_mov_b32 v1, s21
; GFX12-GISEL-TRUE16-NEXT: v_dual_mov_b32 v2, s22 :: v_dual_mov_b32 v3, s4
; GFX12-GISEL-TRUE16-NEXT: v_dual_mov_b32 v6, s0 :: v_dual_mov_b32 v7, s1
@@ -258,12 +260,14 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16(i64 inreg %node_ptr,
;
; GFX12-GISEL-TRUE16-LABEL: image_bvh64_intersect_ray_a16:
; GFX12-GISEL-TRUE16: ; %bb.0: ; %main_body
-; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s21, s4
-; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s4, s8, s6
; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s20, s3
; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s22, s5
+; GFX12-GISEL-TRUE16-NEXT: s_lshr_b32 s3, s6, 16
+; GFX12-GISEL-TRUE16-NEXT: s_lshr_b32 s5, s8, 16
+; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s21, s4
+; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s4, s8, s6
+; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s5, s3
; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s6, s9, s7
-; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-GISEL-TRUE16-NEXT: v_dual_mov_b32 v0, s20 :: v_dual_mov_b32 v3, s4
; GFX12-GISEL-TRUE16-NEXT: v_dual_mov_b32 v7, s1 :: v_dual_mov_b32 v6, s0
; GFX12-GISEL-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s21
diff --git a/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir b/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir
index 176d7752133cf..ac25b6a760bf0 100644
--- a/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir
+++ b/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir
@@ -210,21 +210,6 @@ body: |
...
----
-name: fold_sreg_64_hi16_to_sgpr_lo16
-body: |
- bb.0:
-
- ; GCN-LABEL: name: fold_sreg_64_hi16_to_sgpr_lo16
- ; GCN: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 1125912791875585
- ; GCN-NEXT: $sgpr0 = S_MOV_B32 2
- ; GCN-NEXT: SI_RETURN_TO_EPILOG $sgpr0_lo16
- %0:sreg_64 = S_MOV_B64_IMM_PSEUDO 1125912791875585
- $sgpr0_lo16 = COPY killed %0.hi16
- SI_RETURN_TO_EPILOG $sgpr0_lo16
-
-...
-
---
name: fold_sreg_64_sub1_lo16_to_sgpr_lo16
body: |
@@ -820,20 +805,6 @@ body: |
%1:sgpr_lo16 = COPY killed %0.lo16
SI_RETURN_TO_EPILOG %1
-...
----
-name: fold_simm_16_sub_to_hi_from_mov_64_inline_imm_virt_sgpr16
-body: |
- bb.0:
-
- ; GCN-LABEL: name: fold_simm_16_sub_to_hi_from_mov_64_inline_imm_virt_sgpr16
- ; GCN: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 64
- ; GCN-NEXT: [[COPY:%[0-9]+]]:sgpr_lo16 = COPY killed [[S_MOV_B64_]].hi16
- ; GCN-NEXT: SI_RETURN_TO_EPILOG [[COPY]]
- %0:sreg_64 = S_MOV_B64 64
- %1:sgpr_lo16 = COPY killed %0.hi16
- SI_RETURN_TO_EPILOG %1
-
...
---
@@ -850,17 +821,3 @@ body: |
SI_RETURN_TO_EPILOG $sgpr0_lo16
...
----
-name: fold_simm_16_sub_to_hi_from_mov_64_inline_imm_phys_sgpr16_lo
-body: |
- bb.0:
-
- ; GCN-LABEL: name: fold_simm_16_sub_to_hi_from_mov_64_inline_imm_phys_sgpr16_lo
- ; GCN: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 64
- ; GCN-NEXT: $sgpr0 = S_MOV_B32 0
- ; GCN-NEXT: SI_RETURN_TO_EPILOG $sgpr0_lo16
- %0:sreg_64 = S_MOV_B64 64
- $sgpr0_lo16 = COPY killed %0.hi16
- SI_RETURN_TO_EPILOG $sgpr0_lo16
-
-...
>From 0aeeb9279c5ba0385e9cbfbc94b93219b7a59b70 Mon Sep 17 00:00:00 2001
From: Igor Wodiany <igor.wodiany at amd.com>
Date: Thu, 26 Mar 2026 16:56:34 +0000
Subject: [PATCH 2/9] Replace reg fix
---
llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
index 4b629534caa6c..23b8145c79464 100644
--- a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
@@ -245,7 +245,7 @@ bool InstructionSelect::selectMachineFunction(MachineFunction &MF) {
Register DstReg = MI.getOperand(0).getReg();
unsigned SrcSubIdx = MI.getOperand(1).getSubReg();
unsigned DstSubIdx = MI.getOperand(0).getSubReg();
- if (!SrcReg.isVirtual() || !DstReg.isVirtual() || SrcSubIdx != DstSubIdx)
+ if (!SrcReg.isVirtual() || !DstReg.isVirtual() || SrcSubIdx || DstSubIdx)
continue;
const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);
>From b6e5a4128dcd7ddc1068cce74fbdb4f9385eff3c Mon Sep 17 00:00:00 2001
From: Igor Wodiany <igor.wodiany at amd.com>
Date: Fri, 27 Mar 2026 11:34:16 +0000
Subject: [PATCH 3/9] AArch64 fix
---
.../AArch64/GISel/AArch64InstructionSelector.cpp | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 2fa0fca176c88..087d8103eac9d 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -1105,6 +1105,17 @@ static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
return selectCopy(I, TII, MRI, TRI, RBI);
}
+ // Remove degenerate cases that fail MachineVerifier:
+ //
+ // %0:gpr32all = COPY %1.sub_32:gpr32all
+ //
+ // In this case we can simply use gpr32all without sub_32, so remove
+ // sub-register.
+ MachineOperand &MO = I.getOperand(1);
+ if (RBI.getSizeInBits(MO.getReg(), MRI, TRI) == 32 &&
+ MO.getSubReg() == AArch64::sub_32)
+ I.getOperand(1).setSubReg(0);
+
I.setDesc(TII.get(AArch64::COPY));
return true;
}
>From 0b6e008abdde2a93d080088d10c385bdb771554c Mon Sep 17 00:00:00 2001
From: Igor Wodiany <igor.wodiany at amd.com>
Date: Fri, 27 Mar 2026 13:44:19 +0000
Subject: [PATCH 4/9] Remove AArch64 fix and update propagation
---
llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp | 10 ++++++----
.../AArch64/GISel/AArch64InstructionSelector.cpp | 11 -----------
2 files changed, 6 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
index 23b8145c79464..f05b8cc8f32ca 100644
--- a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
@@ -243,14 +243,16 @@ bool InstructionSelect::selectMachineFunction(MachineFunction &MF) {
continue;
Register SrcReg = MI.getOperand(1).getReg();
Register DstReg = MI.getOperand(0).getReg();
- unsigned SrcSubIdx = MI.getOperand(1).getSubReg();
- unsigned DstSubIdx = MI.getOperand(0).getSubReg();
- if (!SrcReg.isVirtual() || !DstReg.isVirtual() || SrcSubIdx || DstSubIdx)
+ if (!SrcReg.isVirtual() || !DstReg.isVirtual())
continue;
+ const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);
const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
- if (SrcRC == DstRC) {
+ unsigned SubSrcReg = MI.getOperand(1).getSubReg();
+
+ if (SrcRC == DstRC && (!SubSrcReg || TRI.getSubRegIdxSize(SubSrcReg) ==
+ TRI.getRegSizeInBits(*DstRC))) {
MRI.replaceRegWith(DstReg, SrcReg);
MI.eraseFromParent();
}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 087d8103eac9d..2fa0fca176c88 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -1105,17 +1105,6 @@ static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
return selectCopy(I, TII, MRI, TRI, RBI);
}
- // Remove degenerate cases that fail MachineVerifier:
- //
- // %0:gpr32all = COPY %1.sub_32:gpr32all
- //
- // In this case we can simply use gpr32all without sub_32, so remove
- // sub-register.
- MachineOperand &MO = I.getOperand(1);
- if (RBI.getSizeInBits(MO.getReg(), MRI, TRI) == 32 &&
- MO.getSubReg() == AArch64::sub_32)
- I.getOperand(1).setSubReg(0);
-
I.setDesc(TII.get(AArch64::COPY));
return true;
}
>From f7c6b2c4478eebcbed41d45e21747225d7cf709d Mon Sep 17 00:00:00 2001
From: Igor Wodiany <igor.wodiany at amd.com>
Date: Tue, 7 Apr 2026 11:50:20 +0100
Subject: [PATCH 5/9] Revert "Remove AArch64 fix and update propagation"
This reverts commit 0b6e008abdde2a93d080088d10c385bdb771554c.
---
llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp | 10 ++++------
.../AArch64/GISel/AArch64InstructionSelector.cpp | 11 +++++++++++
2 files changed, 15 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
index f05b8cc8f32ca..23b8145c79464 100644
--- a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
@@ -243,16 +243,14 @@ bool InstructionSelect::selectMachineFunction(MachineFunction &MF) {
continue;
Register SrcReg = MI.getOperand(1).getReg();
Register DstReg = MI.getOperand(0).getReg();
- if (!SrcReg.isVirtual() || !DstReg.isVirtual())
+ unsigned SrcSubIdx = MI.getOperand(1).getSubReg();
+ unsigned DstSubIdx = MI.getOperand(0).getSubReg();
+ if (!SrcReg.isVirtual() || !DstReg.isVirtual() || SrcSubIdx || DstSubIdx)
continue;
- const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);
const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
- unsigned SubSrcReg = MI.getOperand(1).getSubReg();
-
- if (SrcRC == DstRC && (!SubSrcReg || TRI.getSubRegIdxSize(SubSrcReg) ==
- TRI.getRegSizeInBits(*DstRC))) {
+ if (SrcRC == DstRC) {
MRI.replaceRegWith(DstReg, SrcReg);
MI.eraseFromParent();
}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 2fa0fca176c88..087d8103eac9d 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -1105,6 +1105,17 @@ static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
return selectCopy(I, TII, MRI, TRI, RBI);
}
+ // Remove degenerate cases that fail MachineVerifier:
+ //
+ // %0:gpr32all = COPY %1.sub_32:gpr32all
+ //
+ // In this case we can simply use gpr32all without sub_32, so remove
+ // sub-register.
+ MachineOperand &MO = I.getOperand(1);
+ if (RBI.getSizeInBits(MO.getReg(), MRI, TRI) == 32 &&
+ MO.getSubReg() == AArch64::sub_32)
+ I.getOperand(1).setSubReg(0);
+
I.setDesc(TII.get(AArch64::COPY));
return true;
}
>From cff5f37105dac81f76eadc5af5f6b0ff67baa3cd Mon Sep 17 00:00:00 2001
From: Igor Wodiany <igor.wodiany at amd.com>
Date: Tue, 7 Apr 2026 11:50:57 +0100
Subject: [PATCH 6/9] Revert "AArch64 fix"
This reverts commit b6e5a4128dcd7ddc1068cce74fbdb4f9385eff3c.
---
.../AArch64/GISel/AArch64InstructionSelector.cpp | 11 -----------
1 file changed, 11 deletions(-)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 087d8103eac9d..2fa0fca176c88 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -1105,17 +1105,6 @@ static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
return selectCopy(I, TII, MRI, TRI, RBI);
}
- // Remove degenerate cases that fail MachineVerifier:
- //
- // %0:gpr32all = COPY %1.sub_32:gpr32all
- //
- // In this case we can simply use gpr32all without sub_32, so remove
- // sub-register.
- MachineOperand &MO = I.getOperand(1);
- if (RBI.getSizeInBits(MO.getReg(), MRI, TRI) == 32 &&
- MO.getSubReg() == AArch64::sub_32)
- I.getOperand(1).setSubReg(0);
-
I.setDesc(TII.get(AArch64::COPY));
return true;
}
>From 91c3fd34975ce653fd819dcccc8171b20982cff1 Mon Sep 17 00:00:00 2001
From: Igor Wodiany <igor.wodiany at amd.com>
Date: Tue, 7 Apr 2026 15:57:53 +0100
Subject: [PATCH 7/9] AArch64 fix
---
llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp | 3 +--
.../Target/AArch64/GISel/AArch64InstructionSelector.cpp | 7 +++++--
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
index 23b8145c79464..fbeddac4890ee 100644
--- a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
@@ -244,8 +244,7 @@ bool InstructionSelect::selectMachineFunction(MachineFunction &MF) {
Register SrcReg = MI.getOperand(1).getReg();
Register DstReg = MI.getOperand(0).getReg();
unsigned SrcSubIdx = MI.getOperand(1).getSubReg();
- unsigned DstSubIdx = MI.getOperand(0).getSubReg();
- if (!SrcReg.isVirtual() || !DstReg.isVirtual() || SrcSubIdx || DstSubIdx)
+ if (!SrcReg.isVirtual() || !DstReg.isVirtual() || SrcSubIdx)
continue;
const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 2fa0fca176c88..ab3df274e0329 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -1024,6 +1024,7 @@ static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
const RegisterBankInfo &RBI) {
Register DstReg = I.getOperand(0).getReg();
Register SrcReg = I.getOperand(1).getReg();
+ Register SubSrcReg = I.getOperand(1).getSubReg();
const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
@@ -1060,9 +1061,11 @@ static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
MachineIRBuilder MIB(I);
auto Copy = MIB.buildCopy({DstTempRC}, {SrcReg});
copySubReg(I, MRI, RBI, Copy.getReg(0), DstRC, SubReg);
- } else if (SrcSize > DstSize) {
+ } else if (SrcSize > DstSize &&
+ (!SubSrcReg || DstSize != TRI.getSubRegIdxSize(SubSrcReg))) {
// If the source register is bigger than the destination we need to
- // perform a subregister copy.
+ // perform a subregister copy, unless there is already a sub-register of a
+ // correct size present.
const TargetRegisterClass *SubRegRC =
getMinClassForRegBank(SrcRegBank, DstSize, /* GetAllRegSet */ true);
getSubRegForClass(SubRegRC, TRI, SubReg);
>From 6a9a760558e7f27b94346421c4c042e04f9f1712 Mon Sep 17 00:00:00 2001
From: Igor Wodiany <igor.wodiany at amd.com>
Date: Fri, 10 Apr 2026 18:00:31 +0100
Subject: [PATCH 8/9] Remove verification
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 29 +------------
.../test/CodeGen/AMDGPU/peephole-fold-imm.mir | 43 +++++++++++++++++++
2 files changed, 45 insertions(+), 27 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 254dab030f25b..df2700d414893 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -5163,19 +5163,6 @@ static bool isSubRegOf(const SIRegisterInfo &TRI,
SubReg.getReg() == SuperVec.getReg();
}
-/// Check if given MachineOperand is an SGPR with SubReg hi16.
-static bool isSGPRHi16(const MachineRegisterInfo &MRI, const SIRegisterInfo &RI,
- const MachineOperand &MO) {
- if (!MO.isReg())
- return false;
-
- Register Reg = MO.getReg();
- if (!Reg || !Reg.isVirtual() || !MRI.getRegClassOrNull(Reg))
- return false;
-
- return RI.isSGPRReg(MRI, Reg) && MO.getSubReg() == AMDGPU::hi16;
-}
-
// Verify the illegal copy from vector register to SGPR for generic opcode COPY
bool SIInstrInfo::verifyCopy(const MachineInstr &MI,
const MachineRegisterInfo &MRI,
@@ -5205,20 +5192,7 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
if (!MRI.isSSA() && MI.isCopy())
return verifyCopy(MI, MRI, ErrInfo);
- if (SIInstrInfo::isGenericOpcode(Opcode) && !MI.isCopy())
- return true;
-
- // Verify that none of the operands in either AMDGPU instruction or COPY
- // is SGPR hi16.
- const MCInstrDesc &Desc = get(Opcode);
- for (int I = 0, E = Desc.getNumOperands(); I != E; ++I) {
- if (isSGPRHi16(MRI, RI, MI.getOperand(I))) {
- ErrInfo = "SGPR cannot use hi16 subreg";
- return false;
- }
- }
-
- if (MI.isCopy())
+ if (SIInstrInfo::isGenericOpcode(Opcode))
return true;
int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
@@ -5234,6 +5208,7 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
}
// Make sure the number of operands is correct.
+ const MCInstrDesc &Desc = get(Opcode);
if (!Desc.isVariadic() &&
Desc.getNumOperands() != MI.getNumExplicitOperands()) {
ErrInfo = "Instruction has wrong number of operands.";
diff --git a/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir b/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir
index ac25b6a760bf0..176d7752133cf 100644
--- a/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir
+++ b/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir
@@ -210,6 +210,21 @@ body: |
...
+---
+name: fold_sreg_64_hi16_to_sgpr_lo16
+body: |
+ bb.0:
+
+ ; GCN-LABEL: name: fold_sreg_64_hi16_to_sgpr_lo16
+ ; GCN: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 1125912791875585
+ ; GCN-NEXT: $sgpr0 = S_MOV_B32 2
+ ; GCN-NEXT: SI_RETURN_TO_EPILOG $sgpr0_lo16
+ %0:sreg_64 = S_MOV_B64_IMM_PSEUDO 1125912791875585
+ $sgpr0_lo16 = COPY killed %0.hi16
+ SI_RETURN_TO_EPILOG $sgpr0_lo16
+
+...
+
---
name: fold_sreg_64_sub1_lo16_to_sgpr_lo16
body: |
@@ -805,6 +820,20 @@ body: |
%1:sgpr_lo16 = COPY killed %0.lo16
SI_RETURN_TO_EPILOG %1
+...
+---
+name: fold_simm_16_sub_to_hi_from_mov_64_inline_imm_virt_sgpr16
+body: |
+ bb.0:
+
+ ; GCN-LABEL: name: fold_simm_16_sub_to_hi_from_mov_64_inline_imm_virt_sgpr16
+ ; GCN: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 64
+ ; GCN-NEXT: [[COPY:%[0-9]+]]:sgpr_lo16 = COPY killed [[S_MOV_B64_]].hi16
+ ; GCN-NEXT: SI_RETURN_TO_EPILOG [[COPY]]
+ %0:sreg_64 = S_MOV_B64 64
+ %1:sgpr_lo16 = COPY killed %0.hi16
+ SI_RETURN_TO_EPILOG %1
+
...
---
@@ -821,3 +850,17 @@ body: |
SI_RETURN_TO_EPILOG $sgpr0_lo16
...
+---
+name: fold_simm_16_sub_to_hi_from_mov_64_inline_imm_phys_sgpr16_lo
+body: |
+ bb.0:
+
+ ; GCN-LABEL: name: fold_simm_16_sub_to_hi_from_mov_64_inline_imm_phys_sgpr16_lo
+ ; GCN: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 64
+ ; GCN-NEXT: $sgpr0 = S_MOV_B32 0
+ ; GCN-NEXT: SI_RETURN_TO_EPILOG $sgpr0_lo16
+ %0:sreg_64 = S_MOV_B64 64
+ $sgpr0_lo16 = COPY killed %0.hi16
+ SI_RETURN_TO_EPILOG $sgpr0_lo16
+
+...
>From 03017c62df3a7f15c82768840fbb37d0983592da Mon Sep 17 00:00:00 2001
From: Igor Wodiany <igor.wodiany at amd.com>
Date: Fri, 10 Apr 2026 18:02:40 +0100
Subject: [PATCH 9/9] Add braces
---
llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index cd0a78660dac0..6c39bd7038394 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -731,13 +731,15 @@ bool AMDGPUInstructionSelector::selectG_UNMERGE_VALUES(MachineInstr &MI) const {
for (int I = 0, E = NumDst; I != E; ++I) {
MachineOperand &Dst = MI.getOperand(I);
// hi16:sreg_32 is not allowed so explicitly shift upper 16-bits.
- if (SrcBank->getID() == AMDGPU::SGPRRegBankID && SubRegs[I] == AMDGPU::hi16)
+ if (SrcBank->getID() == AMDGPU::SGPRRegBankID &&
+ SubRegs[I] == AMDGPU::hi16) {
BuildMI(*BB, &MI, DL, TII.get(AMDGPU::S_LSHR_B32), Dst.getReg())
.addReg(SrcReg)
.addImm(16);
- else
+ } else {
BuildMI(*BB, &MI, DL, TII.get(TargetOpcode::COPY), Dst.getReg())
.addReg(SrcReg, {}, SubRegs[I]);
+ }
// Make sure the subregister index is valid for the source register.
SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubRegs[I]);
More information about the llvm-commits
mailing list