[llvm] a1d7da0 - [AMDGPU][SILoadStoreOptimizer] Merge constrained sloads (#96162)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 23 01:20:47 PDT 2024
Author: Christudasan Devadasan
Date: 2024-07-23T13:50:42+05:30
New Revision: a1d7da05d0e8db4690d605e0c767f30e7891ceb2
URL: https://github.com/llvm/llvm-project/commit/a1d7da05d0e8db4690d605e0c767f30e7891ceb2
DIFF: https://github.com/llvm/llvm-project/commit/a1d7da05d0e8db4690d605e0c767f30e7891ceb2.diff
LOG: [AMDGPU][SILoadStoreOptimizer] Merge constrained sloads (#96162)
Consider the constrained multi-dword loads while merging
individual loads to a single multi-dword load.
Added:
Modified:
llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.update.dpp.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
llvm/test/CodeGen/AMDGPU/add.v2i16.ll
llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
llvm/test/CodeGen/AMDGPU/build_vector.ll
llvm/test/CodeGen/AMDGPU/cluster_stores.ll
llvm/test/CodeGen/AMDGPU/combine-cond-add-sub.ll
llvm/test/CodeGen/AMDGPU/ctlz.ll
llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
llvm/test/CodeGen/AMDGPU/cttz.ll
llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
llvm/test/CodeGen/AMDGPU/fdiv.ll
llvm/test/CodeGen/AMDGPU/flat_atomics.ll
llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll
llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll
llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-atomics.ll
llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-ptr-atomics.ll
llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-atomics.ll
llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-ptr-atomics.ll
llvm/test/CodeGen/AMDGPU/fshl.ll
llvm/test/CodeGen/AMDGPU/fshr.ll
llvm/test/CodeGen/AMDGPU/global_atomics.ll
llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.atomic.csub.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
llvm/test/CodeGen/AMDGPU/llvm.exp.ll
llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
llvm/test/CodeGen/AMDGPU/llvm.log2.ll
llvm/test/CodeGen/AMDGPU/llvm.round.ll
llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll
llvm/test/CodeGen/AMDGPU/madak.ll
llvm/test/CodeGen/AMDGPU/memory_clause.ll
llvm/test/CodeGen/AMDGPU/merge-s-load.mir
llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll
llvm/test/CodeGen/AMDGPU/mul_int24.ll
llvm/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll
llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
llvm/test/CodeGen/AMDGPU/ptr-buffer-alias-scheduling.ll
llvm/test/CodeGen/AMDGPU/rotl.ll
llvm/test/CodeGen/AMDGPU/rotr.ll
llvm/test/CodeGen/AMDGPU/shl.v2i16.ll
llvm/test/CodeGen/AMDGPU/sub.ll
llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
llvm/test/CodeGen/AMDGPU/v_cndmask.ll
llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
llvm/test/CodeGen/AMDGPU/wave32.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
index 8b42d4a1dee7a..ae537b194f50c 100644
--- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
@@ -216,7 +216,7 @@ class SILoadStoreOptimizer : public MachineFunctionPass {
CombineInfo &Paired, bool Modify = false);
static bool widthsFit(const GCNSubtarget &STI, const CombineInfo &CI,
const CombineInfo &Paired);
- static unsigned getNewOpcode(const CombineInfo &CI, const CombineInfo &Paired);
+ unsigned getNewOpcode(const CombineInfo &CI, const CombineInfo &Paired);
static std::pair<unsigned, unsigned> getSubRegIdxs(const CombineInfo &CI,
const CombineInfo &Paired);
const TargetRegisterClass *
@@ -353,6 +353,7 @@ static unsigned getOpcodeWidth(const MachineInstr &MI, const SIInstrInfo &TII) {
case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
case AMDGPU::S_LOAD_DWORDX2_IMM:
+ case AMDGPU::S_LOAD_DWORDX2_IMM_ec:
case AMDGPU::GLOBAL_LOAD_DWORDX2:
case AMDGPU::GLOBAL_LOAD_DWORDX2_SADDR:
case AMDGPU::GLOBAL_STORE_DWORDX2:
@@ -363,6 +364,7 @@ static unsigned getOpcodeWidth(const MachineInstr &MI, const SIInstrInfo &TII) {
case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM:
case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM:
case AMDGPU::S_LOAD_DWORDX3_IMM:
+ case AMDGPU::S_LOAD_DWORDX3_IMM_ec:
case AMDGPU::GLOBAL_LOAD_DWORDX3:
case AMDGPU::GLOBAL_LOAD_DWORDX3_SADDR:
case AMDGPU::GLOBAL_STORE_DWORDX3:
@@ -373,6 +375,7 @@ static unsigned getOpcodeWidth(const MachineInstr &MI, const SIInstrInfo &TII) {
case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
case AMDGPU::S_LOAD_DWORDX4_IMM:
+ case AMDGPU::S_LOAD_DWORDX4_IMM_ec:
case AMDGPU::GLOBAL_LOAD_DWORDX4:
case AMDGPU::GLOBAL_LOAD_DWORDX4_SADDR:
case AMDGPU::GLOBAL_STORE_DWORDX4:
@@ -383,6 +386,7 @@ static unsigned getOpcodeWidth(const MachineInstr &MI, const SIInstrInfo &TII) {
case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM:
case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM:
case AMDGPU::S_LOAD_DWORDX8_IMM:
+ case AMDGPU::S_LOAD_DWORDX8_IMM_ec:
return 8;
case AMDGPU::DS_READ_B32:
case AMDGPU::DS_READ_B32_gfx9:
@@ -507,6 +511,10 @@ static InstClassEnum getInstClass(unsigned Opc, const SIInstrInfo &TII) {
case AMDGPU::S_LOAD_DWORDX3_IMM:
case AMDGPU::S_LOAD_DWORDX4_IMM:
case AMDGPU::S_LOAD_DWORDX8_IMM:
+ case AMDGPU::S_LOAD_DWORDX2_IMM_ec:
+ case AMDGPU::S_LOAD_DWORDX3_IMM_ec:
+ case AMDGPU::S_LOAD_DWORDX4_IMM_ec:
+ case AMDGPU::S_LOAD_DWORDX8_IMM_ec:
return S_LOAD_IMM;
case AMDGPU::DS_READ_B32:
case AMDGPU::DS_READ_B32_gfx9:
@@ -591,6 +599,10 @@ static unsigned getInstSubclass(unsigned Opc, const SIInstrInfo &TII) {
case AMDGPU::S_LOAD_DWORDX3_IMM:
case AMDGPU::S_LOAD_DWORDX4_IMM:
case AMDGPU::S_LOAD_DWORDX8_IMM:
+ case AMDGPU::S_LOAD_DWORDX2_IMM_ec:
+ case AMDGPU::S_LOAD_DWORDX3_IMM_ec:
+ case AMDGPU::S_LOAD_DWORDX4_IMM_ec:
+ case AMDGPU::S_LOAD_DWORDX8_IMM_ec:
return AMDGPU::S_LOAD_DWORD_IMM;
case AMDGPU::GLOBAL_LOAD_DWORD:
case AMDGPU::GLOBAL_LOAD_DWORDX2:
@@ -703,6 +715,10 @@ static AddressRegs getRegs(unsigned Opc, const SIInstrInfo &TII) {
case AMDGPU::S_LOAD_DWORDX3_IMM:
case AMDGPU::S_LOAD_DWORDX4_IMM:
case AMDGPU::S_LOAD_DWORDX8_IMM:
+ case AMDGPU::S_LOAD_DWORDX2_IMM_ec:
+ case AMDGPU::S_LOAD_DWORDX3_IMM_ec:
+ case AMDGPU::S_LOAD_DWORDX4_IMM_ec:
+ case AMDGPU::S_LOAD_DWORDX8_IMM_ec:
Result.SBase = true;
return Result;
case AMDGPU::DS_READ_B32:
@@ -1212,8 +1228,14 @@ void SILoadStoreOptimizer::copyToDestRegs(
// Copy to the old destination registers.
const MCInstrDesc &CopyDesc = TII->get(TargetOpcode::COPY);
- const auto *Dest0 = TII->getNamedOperand(*CI.I, OpName);
- const auto *Dest1 = TII->getNamedOperand(*Paired.I, OpName);
+ auto *Dest0 = TII->getNamedOperand(*CI.I, OpName);
+ auto *Dest1 = TII->getNamedOperand(*Paired.I, OpName);
+
+ // The constrained sload instructions in S_LOAD_IMM class will have
+ // `early-clobber` flag in the dst operand. Remove the flag before using the
+ // MOs in copies.
+ Dest0->setIsEarlyClobber(false);
+ Dest1->setIsEarlyClobber(false);
BuildMI(*MBB, InsertBefore, DL, CopyDesc)
.add(*Dest0) // Copy to same destination including flags and sub reg.
@@ -1700,19 +1722,29 @@ unsigned SILoadStoreOptimizer::getNewOpcode(const CombineInfo &CI,
case 8:
return AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM;
}
- case S_LOAD_IMM:
+ case S_LOAD_IMM: {
+ // If XNACK is enabled, use the constrained opcodes when the first load is
+ // under-aligned.
+ const MachineMemOperand *MMO = *CI.I->memoperands_begin();
+ bool NeedsConstrainedOpc =
+ STM->isXNACKEnabled() && MMO->getAlign().value() < Width * 4;
switch (Width) {
default:
return 0;
case 2:
- return AMDGPU::S_LOAD_DWORDX2_IMM;
+ return NeedsConstrainedOpc ? AMDGPU::S_LOAD_DWORDX2_IMM_ec
+ : AMDGPU::S_LOAD_DWORDX2_IMM;
case 3:
- return AMDGPU::S_LOAD_DWORDX3_IMM;
+ return NeedsConstrainedOpc ? AMDGPU::S_LOAD_DWORDX3_IMM_ec
+ : AMDGPU::S_LOAD_DWORDX3_IMM;
case 4:
- return AMDGPU::S_LOAD_DWORDX4_IMM;
+ return NeedsConstrainedOpc ? AMDGPU::S_LOAD_DWORDX4_IMM_ec
+ : AMDGPU::S_LOAD_DWORDX4_IMM;
case 8:
- return AMDGPU::S_LOAD_DWORDX8_IMM;
+ return NeedsConstrainedOpc ? AMDGPU::S_LOAD_DWORDX8_IMM_ec
+ : AMDGPU::S_LOAD_DWORDX8_IMM;
}
+ }
case GLOBAL_LOAD:
switch (Width) {
default:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
index 69f9a5712b0b5..d1fa579ca9b4b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
@@ -658,17 +658,17 @@ define amdgpu_kernel void @image_bvh_intersect_ray_nsa_reassign(ptr %p_node_ptr,
;
; GFX1013-LABEL: image_bvh_intersect_ray_nsa_reassign:
; GFX1013: ; %bb.0:
-; GFX1013-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
+; GFX1013-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; GFX1013-NEXT: v_lshlrev_b32_e32 v6, 2, v0
; GFX1013-NEXT: v_mov_b32_e32 v7, 0x40a00000
; GFX1013-NEXT: v_mov_b32_e32 v8, 0x40c00000
; GFX1013-NEXT: v_mov_b32_e32 v9, 0x40e00000
; GFX1013-NEXT: v_mov_b32_e32 v10, 0x41000000
; GFX1013-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1013-NEXT: v_mov_b32_e32 v0, s0
-; GFX1013-NEXT: v_mov_b32_e32 v1, s1
-; GFX1013-NEXT: v_mov_b32_e32 v2, s2
-; GFX1013-NEXT: v_mov_b32_e32 v3, s3
+; GFX1013-NEXT: v_mov_b32_e32 v0, s4
+; GFX1013-NEXT: v_mov_b32_e32 v1, s5
+; GFX1013-NEXT: v_mov_b32_e32 v2, s6
+; GFX1013-NEXT: v_mov_b32_e32 v3, s7
; GFX1013-NEXT: v_add_co_u32 v4, vcc_lo, v0, v6
; GFX1013-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo
; GFX1013-NEXT: v_add_co_u32 v2, vcc_lo, v2, v6
@@ -681,7 +681,7 @@ define amdgpu_kernel void @image_bvh_intersect_ray_nsa_reassign(ptr %p_node_ptr,
; GFX1013-NEXT: v_mov_b32_e32 v4, 2.0
; GFX1013-NEXT: v_mov_b32_e32 v5, 0x40400000
; GFX1013-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX1013-NEXT: image_bvh_intersect_ray v[0:3], v[0:10], s[4:7]
+; GFX1013-NEXT: image_bvh_intersect_ray v[0:3], v[0:10], s[8:11]
; GFX1013-NEXT: s_waitcnt vmcnt(0)
; GFX1013-NEXT: flat_store_dwordx4 v[0:1], v[0:3]
; GFX1013-NEXT: s_endpgm
@@ -769,14 +769,14 @@ define amdgpu_kernel void @image_bvh_intersect_ray_a16_nsa_reassign(ptr %p_node_
;
; GFX1013-LABEL: image_bvh_intersect_ray_a16_nsa_reassign:
; GFX1013: ; %bb.0:
-; GFX1013-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
+; GFX1013-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; GFX1013-NEXT: v_lshlrev_b32_e32 v6, 2, v0
; GFX1013-NEXT: v_mov_b32_e32 v7, 0x48004700
; GFX1013-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1013-NEXT: v_mov_b32_e32 v0, s0
-; GFX1013-NEXT: v_mov_b32_e32 v1, s1
-; GFX1013-NEXT: v_mov_b32_e32 v2, s2
-; GFX1013-NEXT: v_mov_b32_e32 v3, s3
+; GFX1013-NEXT: v_mov_b32_e32 v0, s4
+; GFX1013-NEXT: v_mov_b32_e32 v1, s5
+; GFX1013-NEXT: v_mov_b32_e32 v2, s6
+; GFX1013-NEXT: v_mov_b32_e32 v3, s7
; GFX1013-NEXT: v_add_co_u32 v4, vcc_lo, v0, v6
; GFX1013-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo
; GFX1013-NEXT: v_add_co_u32 v2, vcc_lo, v2, v6
@@ -789,7 +789,7 @@ define amdgpu_kernel void @image_bvh_intersect_ray_a16_nsa_reassign(ptr %p_node_
; GFX1013-NEXT: v_mov_b32_e32 v4, 2.0
; GFX1013-NEXT: v_mov_b32_e32 v5, 0x44004200
; GFX1013-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX1013-NEXT: image_bvh_intersect_ray v[0:3], v[0:7], s[4:7] a16
+; GFX1013-NEXT: image_bvh_intersect_ray v[0:3], v[0:7], s[8:11] a16
; GFX1013-NEXT: s_waitcnt vmcnt(0)
; GFX1013-NEXT: flat_store_dwordx4 v[0:1], v[0:3]
; GFX1013-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.update.dpp.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.update.dpp.ll
index 2198ba9f1d964..746bd739644a9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.update.dpp.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.update.dpp.ll
@@ -19,13 +19,13 @@ define amdgpu_kernel void @dpp_test(ptr addrspace(1) %out, i32 %in1, i32 %in2) {
;
; GFX10-LABEL: dpp_test:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-NEXT: v_mov_b32_e32 v1, s7
; GFX10-NEXT: v_mov_b32_dpp v0, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: dpp_test:
@@ -176,16 +176,16 @@ define amdgpu_kernel void @update_dppv2i32_test(ptr addrspace(1) %arg, <2 x i32>
;
; GFX10-LABEL: update_dppv2i32_test:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx2 v[0:1], v4, s[0:1]
-; GFX10-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v4, s[4:5]
+; GFX10-NEXT: v_mov_b32_e32 v2, s6
+; GFX10-NEXT: v_mov_b32_e32 v3, s7
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
; GFX10-NEXT: v_mov_b32_dpp v3, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
-; GFX10-NEXT: global_store_dwordx2 v4, v[2:3], s[0:1]
+; GFX10-NEXT: global_store_dwordx2 v4, v[2:3], s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: update_dppv2i32_test:
@@ -232,16 +232,16 @@ define amdgpu_kernel void @update_dppv2f32_test(ptr addrspace(1) %arg, <2 x floa
;
; GFX10-LABEL: update_dppv2f32_test:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx2 v[0:1], v4, s[0:1]
-; GFX10-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v4, s[4:5]
+; GFX10-NEXT: v_mov_b32_e32 v2, s6
+; GFX10-NEXT: v_mov_b32_e32 v3, s7
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_dpp v2, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
; GFX10-NEXT: v_mov_b32_dpp v3, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
-; GFX10-NEXT: global_store_dwordx2 v4, v[2:3], s[0:1]
+; GFX10-NEXT: global_store_dwordx2 v4, v[2:3], s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: update_dppv2f32_test:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
index cf69c50ed9357..3bc9a582ebd96 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
@@ -692,121 +692,121 @@ define amdgpu_kernel void @sdivrem_v2i32(ptr addrspace(1) %out0, ptr addrspace(1
;
; GFX9-LABEL: sdivrem_v2i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx8 s[0:7], s[6:7], 0x0
+; GFX9-NEXT: s_load_dwordx8 s[8:15], s[6:7], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_ashr_i32 s8, s6, 31
-; GFX9-NEXT: s_add_i32 s6, s6, s8
-; GFX9-NEXT: s_xor_b32 s6, s6, s8
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX9-NEXT: s_ashr_i32 s9, s7, 31
-; GFX9-NEXT: s_add_i32 s7, s7, s9
-; GFX9-NEXT: s_xor_b32 s7, s7, s9
+; GFX9-NEXT: s_ashr_i32 s0, s14, 31
+; GFX9-NEXT: s_add_i32 s1, s14, s0
+; GFX9-NEXT: s_xor_b32 s1, s1, s0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s1
+; GFX9-NEXT: s_ashr_i32 s2, s15, 31
+; GFX9-NEXT: s_add_i32 s3, s15, s2
+; GFX9-NEXT: s_xor_b32 s3, s3, s2
; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s7
-; GFX9-NEXT: s_sub_i32 s12, 0, s6
-; GFX9-NEXT: s_ashr_i32 s10, s4, 31
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s3
+; GFX9-NEXT: s_sub_i32 s6, 0, s1
+; GFX9-NEXT: s_ashr_i32 s4, s12, 31
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1
-; GFX9-NEXT: s_add_i32 s4, s4, s10
-; GFX9-NEXT: s_xor_b32 s4, s4, s10
-; GFX9-NEXT: v_mul_lo_u32 v2, s12, v0
+; GFX9-NEXT: s_sub_i32 s7, 0, s3
+; GFX9-NEXT: s_ashr_i32 s5, s13, 31
+; GFX9-NEXT: v_mul_lo_u32 v2, s6, v0
; GFX9-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT: s_sub_i32 s12, 0, s7
+; GFX9-NEXT: s_add_i32 s6, s12, s4
; GFX9-NEXT: v_mul_hi_u32 v2, v0, v2
-; GFX9-NEXT: s_ashr_i32 s11, s5, 31
-; GFX9-NEXT: v_mul_lo_u32 v3, s12, v1
-; GFX9-NEXT: s_add_i32 s5, s5, s11
+; GFX9-NEXT: s_xor_b32 s6, s6, s4
+; GFX9-NEXT: v_mul_lo_u32 v3, s7, v1
+; GFX9-NEXT: s_add_i32 s7, s13, s5
; GFX9-NEXT: v_add_u32_e32 v0, v0, v2
-; GFX9-NEXT: v_mul_hi_u32 v0, s4, v0
+; GFX9-NEXT: v_mul_hi_u32 v0, s6, v0
; GFX9-NEXT: v_mul_hi_u32 v2, v1, v3
-; GFX9-NEXT: s_xor_b32 s5, s5, s11
-; GFX9-NEXT: v_mul_lo_u32 v3, v0, s6
+; GFX9-NEXT: s_xor_b32 s7, s7, s5
+; GFX9-NEXT: s_xor_b32 s0, s4, s0
+; GFX9-NEXT: v_mul_lo_u32 v3, v0, s1
; GFX9-NEXT: v_add_u32_e32 v1, v1, v2
; GFX9-NEXT: v_add_u32_e32 v2, 1, v0
-; GFX9-NEXT: v_mul_hi_u32 v1, s5, v1
-; GFX9-NEXT: v_sub_u32_e32 v3, s4, v3
-; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s6, v3
+; GFX9-NEXT: v_mul_hi_u32 v1, s7, v1
+; GFX9-NEXT: v_sub_u32_e32 v3, s6, v3
+; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s1, v3
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT: v_subrev_u32_e32 v2, s6, v3
+; GFX9-NEXT: v_subrev_u32_e32 v2, s1, v3
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
; GFX9-NEXT: v_add_u32_e32 v3, 1, v0
-; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s6, v2
+; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s1, v2
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX9-NEXT: v_subrev_u32_e32 v3, s6, v2
+; GFX9-NEXT: v_subrev_u32_e32 v3, s1, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
-; GFX9-NEXT: v_mul_lo_u32 v3, v1, s7
+; GFX9-NEXT: v_mul_lo_u32 v3, v1, s3
; GFX9-NEXT: v_add_u32_e32 v4, 1, v1
-; GFX9-NEXT: s_xor_b32 s4, s10, s8
-; GFX9-NEXT: v_xor_b32_e32 v0, s4, v0
-; GFX9-NEXT: v_sub_u32_e32 v3, s5, v3
-; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s7, v3
+; GFX9-NEXT: v_xor_b32_e32 v0, s0, v0
+; GFX9-NEXT: v_subrev_u32_e32 v0, s0, v0
+; GFX9-NEXT: v_sub_u32_e32 v3, s7, v3
+; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s3, v3
; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT: v_subrev_u32_e32 v4, s7, v3
+; GFX9-NEXT: v_subrev_u32_e32 v4, s3, v3
; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
; GFX9-NEXT: v_add_u32_e32 v4, 1, v1
-; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s7, v3
-; GFX9-NEXT: v_subrev_u32_e32 v0, s4, v0
+; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s3, v3
; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT: v_subrev_u32_e32 v4, s7, v3
-; GFX9-NEXT: s_xor_b32 s4, s11, s9
+; GFX9-NEXT: v_subrev_u32_e32 v4, s3, v3
+; GFX9-NEXT: s_xor_b32 s0, s5, s2
; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX9-NEXT: v_xor_b32_e32 v1, s4, v1
-; GFX9-NEXT: v_xor_b32_e32 v2, s10, v2
-; GFX9-NEXT: v_subrev_u32_e32 v1, s4, v1
-; GFX9-NEXT: v_xor_b32_e32 v3, s11, v3
+; GFX9-NEXT: v_xor_b32_e32 v1, s0, v1
+; GFX9-NEXT: v_xor_b32_e32 v2, s4, v2
+; GFX9-NEXT: v_subrev_u32_e32 v1, s0, v1
+; GFX9-NEXT: v_xor_b32_e32 v3, s5, v3
; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_subrev_u32_e32 v2, s10, v2
-; GFX9-NEXT: v_subrev_u32_e32 v3, s11, v3
-; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
-; GFX9-NEXT: global_store_dwordx2 v4, v[2:3], s[2:3]
+; GFX9-NEXT: v_subrev_u32_e32 v2, s4, v2
+; GFX9-NEXT: v_subrev_u32_e32 v3, s5, v3
+; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[8:9]
+; GFX9-NEXT: global_store_dwordx2 v4, v[2:3], s[10:11]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: sdivrem_v2i32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx8 s[4:11], s[6:7], 0x0
+; GFX10-NEXT: s_load_dwordx8 s[8:15], s[6:7], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_ashr_i32 s1, s10, 31
-; GFX10-NEXT: s_ashr_i32 s2, s11, 31
-; GFX10-NEXT: s_add_i32 s0, s10, s1
-; GFX10-NEXT: s_add_i32 s3, s11, s2
-; GFX10-NEXT: s_xor_b32 s10, s0, s1
+; GFX10-NEXT: s_ashr_i32 s1, s14, 31
+; GFX10-NEXT: s_ashr_i32 s2, s15, 31
+; GFX10-NEXT: s_add_i32 s0, s14, s1
+; GFX10-NEXT: s_add_i32 s3, s15, s2
+; GFX10-NEXT: s_xor_b32 s4, s0, s1
; GFX10-NEXT: s_xor_b32 s3, s3, s2
-; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s10
+; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s4
; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s3
-; GFX10-NEXT: s_sub_i32 s0, 0, s10
-; GFX10-NEXT: s_sub_i32 s11, 0, s3
-; GFX10-NEXT: s_ashr_i32 s12, s9, 31
+; GFX10-NEXT: s_sub_i32 s0, 0, s4
+; GFX10-NEXT: s_sub_i32 s5, 0, s3
+; GFX10-NEXT: s_ashr_i32 s6, s13, 31
; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX10-NEXT: v_rcp_iflag_f32_e32 v1, v1
+; GFX10-NEXT: s_add_i32 s7, s13, s6
+; GFX10-NEXT: s_xor_b32 s7, s7, s6
; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX10-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX10-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX10-NEXT: v_mul_lo_u32 v2, s0, v0
-; GFX10-NEXT: v_mul_lo_u32 v3, s11, v1
-; GFX10-NEXT: s_ashr_i32 s11, s8, 31
-; GFX10-NEXT: s_add_i32 s0, s8, s11
-; GFX10-NEXT: s_add_i32 s8, s9, s12
-; GFX10-NEXT: s_xor_b32 s0, s0, s11
-; GFX10-NEXT: s_xor_b32 s8, s8, s12
+; GFX10-NEXT: v_mul_lo_u32 v3, s5, v1
+; GFX10-NEXT: s_ashr_i32 s5, s12, 31
+; GFX10-NEXT: s_add_i32 s0, s12, s5
+; GFX10-NEXT: s_xor_b32 s1, s5, s1
+; GFX10-NEXT: s_xor_b32 s0, s0, s5
; GFX10-NEXT: v_mul_hi_u32 v2, v0, v2
; GFX10-NEXT: v_mul_hi_u32 v3, v1, v3
-; GFX10-NEXT: s_xor_b32 s1, s11, s1
; GFX10-NEXT: v_add_nc_u32_e32 v0, v0, v2
; GFX10-NEXT: v_add_nc_u32_e32 v1, v1, v3
; GFX10-NEXT: v_mul_hi_u32 v0, s0, v0
-; GFX10-NEXT: v_mul_hi_u32 v1, s8, v1
-; GFX10-NEXT: v_mul_lo_u32 v2, v0, s10
+; GFX10-NEXT: v_mul_hi_u32 v1, s7, v1
+; GFX10-NEXT: v_mul_lo_u32 v2, v0, s4
; GFX10-NEXT: v_mul_lo_u32 v3, v1, s3
; GFX10-NEXT: v_add_nc_u32_e32 v4, 1, v0
; GFX10-NEXT: v_add_nc_u32_e32 v5, 1, v1
; GFX10-NEXT: v_sub_nc_u32_e32 v2, s0, v2
-; GFX10-NEXT: v_sub_nc_u32_e32 v3, s8, v3
-; GFX10-NEXT: v_subrev_nc_u32_e32 v6, s10, v2
+; GFX10-NEXT: v_sub_nc_u32_e32 v3, s7, v3
+; GFX10-NEXT: v_subrev_nc_u32_e32 v6, s4, v2
; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s3, v3
-; GFX10-NEXT: v_cmp_le_u32_e64 s0, s10, v2
+; GFX10-NEXT: v_cmp_le_u32_e64 s0, s4, v2
; GFX10-NEXT: v_subrev_nc_u32_e32 v7, s3, v3
; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
@@ -814,26 +814,26 @@ define amdgpu_kernel void @sdivrem_v2i32(ptr addrspace(1) %out0, ptr addrspace(1
; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
; GFX10-NEXT: v_add_nc_u32_e32 v5, 1, v1
; GFX10-NEXT: v_add_nc_u32_e32 v4, 1, v0
-; GFX10-NEXT: v_cmp_le_u32_e64 s0, s10, v2
+; GFX10-NEXT: v_cmp_le_u32_e64 s0, s4, v2
; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s3, v3
-; GFX10-NEXT: v_subrev_nc_u32_e32 v6, s10, v2
+; GFX10-NEXT: v_subrev_nc_u32_e32 v6, s4, v2
; GFX10-NEXT: v_subrev_nc_u32_e32 v7, s3, v3
; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v6, s0
; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX10-NEXT: s_xor_b32 s0, s12, s2
+; GFX10-NEXT: s_xor_b32 s0, s6, s2
; GFX10-NEXT: v_xor_b32_e32 v0, s1, v0
; GFX10-NEXT: v_xor_b32_e32 v1, s0, v1
-; GFX10-NEXT: v_xor_b32_e32 v2, s11, v2
-; GFX10-NEXT: v_xor_b32_e32 v3, s12, v3
+; GFX10-NEXT: v_xor_b32_e32 v2, s5, v2
+; GFX10-NEXT: v_xor_b32_e32 v3, s6, v3
; GFX10-NEXT: v_mov_b32_e32 v4, 0
; GFX10-NEXT: v_subrev_nc_u32_e32 v0, s1, v0
; GFX10-NEXT: v_subrev_nc_u32_e32 v1, s0, v1
-; GFX10-NEXT: v_subrev_nc_u32_e32 v2, s11, v2
-; GFX10-NEXT: v_subrev_nc_u32_e32 v3, s12, v3
-; GFX10-NEXT: global_store_dwordx2 v4, v[0:1], s[4:5]
-; GFX10-NEXT: global_store_dwordx2 v4, v[2:3], s[6:7]
+; GFX10-NEXT: v_subrev_nc_u32_e32 v2, s5, v2
+; GFX10-NEXT: v_subrev_nc_u32_e32 v3, s6, v3
+; GFX10-NEXT: global_store_dwordx2 v4, v[0:1], s[8:9]
+; GFX10-NEXT: global_store_dwordx2 v4, v[2:3], s[10:11]
; GFX10-NEXT: s_endpgm
%div = sdiv <2 x i32> %x, %y
store <2 x i32> %div, ptr addrspace(1) %out0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
index 8b94f93e44e56..0389cacb61390 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
@@ -576,12 +576,12 @@ define amdgpu_kernel void @udivrem_v2i32(ptr addrspace(1) %out0, ptr addrspace(1
;
; GFX9-LABEL: udivrem_v2i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx8 s[4:11], s[6:7], 0x0
+; GFX9-NEXT: s_load_dwordx8 s[8:15], s[6:7], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s10
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s11
-; GFX9-NEXT: s_sub_i32 s0, 0, s10
-; GFX9-NEXT: s_sub_i32 s1, 0, s11
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s14
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s15
+; GFX9-NEXT: s_sub_i32 s0, 0, s14
+; GFX9-NEXT: s_sub_i32 s1, 0, s15
; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
@@ -593,47 +593,47 @@ define amdgpu_kernel void @udivrem_v2i32(ptr addrspace(1) %out0, ptr addrspace(1
; GFX9-NEXT: v_mul_hi_u32 v2, v0, v2
; GFX9-NEXT: v_mul_hi_u32 v3, v1, v3
; GFX9-NEXT: v_add_u32_e32 v0, v0, v2
-; GFX9-NEXT: v_mul_hi_u32 v0, s8, v0
+; GFX9-NEXT: v_mul_hi_u32 v0, s12, v0
; GFX9-NEXT: v_add_u32_e32 v1, v1, v3
-; GFX9-NEXT: v_mul_hi_u32 v1, s9, v1
-; GFX9-NEXT: v_mul_lo_u32 v2, v0, s10
+; GFX9-NEXT: v_mul_hi_u32 v1, s13, v1
+; GFX9-NEXT: v_mul_lo_u32 v2, v0, s14
; GFX9-NEXT: v_add_u32_e32 v4, 1, v0
-; GFX9-NEXT: v_mul_lo_u32 v3, v1, s11
+; GFX9-NEXT: v_mul_lo_u32 v3, v1, s15
; GFX9-NEXT: v_add_u32_e32 v5, 1, v1
-; GFX9-NEXT: v_sub_u32_e32 v2, s8, v2
-; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s10, v2
-; GFX9-NEXT: v_sub_u32_e32 v3, s9, v3
+; GFX9-NEXT: v_sub_u32_e32 v2, s12, v2
+; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s14, v2
+; GFX9-NEXT: v_sub_u32_e32 v3, s13, v3
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX9-NEXT: v_subrev_u32_e32 v4, s10, v2
-; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s11, v3
+; GFX9-NEXT: v_subrev_u32_e32 v4, s14, v2
+; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s15, v3
; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v5, s[0:1]
-; GFX9-NEXT: v_subrev_u32_e32 v5, s11, v3
+; GFX9-NEXT: v_subrev_u32_e32 v5, s15, v3
; GFX9-NEXT: v_add_u32_e32 v4, 1, v0
-; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s10, v2
+; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s14, v2
; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v5, s[0:1]
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX9-NEXT: v_subrev_u32_e32 v4, s10, v2
+; GFX9-NEXT: v_subrev_u32_e32 v4, s14, v2
; GFX9-NEXT: v_add_u32_e32 v5, 1, v1
; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s11, v3
-; GFX9-NEXT: v_subrev_u32_e32 v4, s11, v3
+; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s15, v3
+; GFX9-NEXT: v_subrev_u32_e32 v4, s15, v3
; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[4:5]
-; GFX9-NEXT: global_store_dwordx2 v4, v[2:3], s[6:7]
+; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[8:9]
+; GFX9-NEXT: global_store_dwordx2 v4, v[2:3], s[10:11]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: udivrem_v2i32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx8 s[4:11], s[6:7], 0x0
+; GFX10-NEXT: s_load_dwordx8 s[8:15], s[6:7], 0x0
; GFX10-NEXT: v_mov_b32_e32 v8, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s10
-; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s11
-; GFX10-NEXT: s_sub_i32 s0, 0, s10
-; GFX10-NEXT: s_sub_i32 s1, 0, s11
+; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s14
+; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s15
+; GFX10-NEXT: s_sub_i32 s0, 0, s14
+; GFX10-NEXT: s_sub_i32 s1, 0, s15
; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX10-NEXT: v_rcp_iflag_f32_e32 v1, v1
; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
@@ -646,34 +646,34 @@ define amdgpu_kernel void @udivrem_v2i32(ptr addrspace(1) %out0, ptr addrspace(1
; GFX10-NEXT: v_mul_hi_u32 v3, v1, v3
; GFX10-NEXT: v_add_nc_u32_e32 v0, v0, v2
; GFX10-NEXT: v_add_nc_u32_e32 v1, v1, v3
-; GFX10-NEXT: v_mul_hi_u32 v0, s8, v0
-; GFX10-NEXT: v_mul_hi_u32 v1, s9, v1
-; GFX10-NEXT: v_mul_lo_u32 v2, v0, s10
-; GFX10-NEXT: v_mul_lo_u32 v3, v1, s11
+; GFX10-NEXT: v_mul_hi_u32 v0, s12, v0
+; GFX10-NEXT: v_mul_hi_u32 v1, s13, v1
+; GFX10-NEXT: v_mul_lo_u32 v2, v0, s14
+; GFX10-NEXT: v_mul_lo_u32 v3, v1, s15
; GFX10-NEXT: v_add_nc_u32_e32 v4, 1, v0
; GFX10-NEXT: v_add_nc_u32_e32 v5, 1, v1
-; GFX10-NEXT: v_sub_nc_u32_e32 v2, s8, v2
-; GFX10-NEXT: v_sub_nc_u32_e32 v3, s9, v3
-; GFX10-NEXT: v_subrev_nc_u32_e32 v6, s10, v2
-; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s11, v3
-; GFX10-NEXT: v_cmp_le_u32_e64 s0, s10, v2
-; GFX10-NEXT: v_subrev_nc_u32_e32 v7, s11, v3
+; GFX10-NEXT: v_sub_nc_u32_e32 v2, s12, v2
+; GFX10-NEXT: v_sub_nc_u32_e32 v3, s13, v3
+; GFX10-NEXT: v_subrev_nc_u32_e32 v6, s14, v2
+; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s15, v3
+; GFX10-NEXT: v_cmp_le_u32_e64 s0, s14, v2
+; GFX10-NEXT: v_subrev_nc_u32_e32 v7, s15, v3
; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v6, s0
; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
; GFX10-NEXT: v_add_nc_u32_e32 v5, 1, v1
; GFX10-NEXT: v_add_nc_u32_e32 v4, 1, v0
-; GFX10-NEXT: v_cmp_le_u32_e64 s0, s10, v2
-; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s11, v3
-; GFX10-NEXT: v_subrev_nc_u32_e32 v6, s10, v2
-; GFX10-NEXT: v_subrev_nc_u32_e32 v7, s11, v3
+; GFX10-NEXT: v_cmp_le_u32_e64 s0, s14, v2
+; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s15, v3
+; GFX10-NEXT: v_subrev_nc_u32_e32 v6, s14, v2
+; GFX10-NEXT: v_subrev_nc_u32_e32 v7, s15, v3
; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v6, s0
; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
-; GFX10-NEXT: global_store_dwordx2 v8, v[0:1], s[4:5]
-; GFX10-NEXT: global_store_dwordx2 v8, v[2:3], s[6:7]
+; GFX10-NEXT: global_store_dwordx2 v8, v[0:1], s[8:9]
+; GFX10-NEXT: global_store_dwordx2 v8, v[2:3], s[10:11]
; GFX10-NEXT: s_endpgm
%div = udiv <2 x i32> %x, %y
store <2 x i32> %div, ptr addrspace(1) %out0
diff --git a/llvm/test/CodeGen/AMDGPU/add.v2i16.ll b/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
index b751be51a9739..3720b9da52dcd 100644
--- a/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
@@ -245,21 +245,21 @@ define amdgpu_kernel void @s_test_add_v2i16_kernarg(ptr addrspace(1) %out, <2 x
;
; GFX9-LABEL: s_test_add_v2i16_kernarg:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: v_pk_add_u16 v1, s2, v1
-; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: v_pk_add_u16 v1, s6, v1
+; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: s_test_add_v2i16_kernarg:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_pk_add_u16 v1, s2, s3
-; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: v_pk_add_u16 v1, s6, s7
+; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_test_add_v2i16_kernarg:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
index 7cf18171a6cd7..cf04dc8e59ead 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
@@ -72,31 +72,31 @@ define amdgpu_kernel void @udiv_i32(ptr addrspace(1) %out, i32 %x, i32 %y) {
;
; GFX9-LABEL: udiv_i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3
-; GFX9-NEXT: s_sub_i32 s4, 0, s3
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7
+; GFX9-NEXT: s_sub_i32 s0, 0, s7
; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: v_readfirstlane_b32 s5, v0
-; GFX9-NEXT: s_mul_i32 s4, s4, s5
-; GFX9-NEXT: s_mul_hi_u32 s4, s5, s4
-; GFX9-NEXT: s_add_i32 s5, s5, s4
-; GFX9-NEXT: s_mul_hi_u32 s4, s2, s5
-; GFX9-NEXT: s_mul_i32 s5, s4, s3
-; GFX9-NEXT: s_sub_i32 s2, s2, s5
-; GFX9-NEXT: s_add_i32 s6, s4, 1
-; GFX9-NEXT: s_sub_i32 s5, s2, s3
-; GFX9-NEXT: s_cmp_ge_u32 s2, s3
-; GFX9-NEXT: s_cselect_b32 s4, s6, s4
-; GFX9-NEXT: s_cselect_b32 s2, s5, s2
-; GFX9-NEXT: s_add_i32 s5, s4, 1
-; GFX9-NEXT: s_cmp_ge_u32 s2, s3
-; GFX9-NEXT: s_cselect_b32 s2, s5, s4
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_mul_i32 s0, s0, s1
+; GFX9-NEXT: s_mul_hi_u32 s0, s1, s0
+; GFX9-NEXT: s_add_i32 s1, s1, s0
+; GFX9-NEXT: s_mul_hi_u32 s0, s6, s1
+; GFX9-NEXT: s_mul_i32 s1, s0, s7
+; GFX9-NEXT: s_sub_i32 s1, s6, s1
+; GFX9-NEXT: s_add_i32 s2, s0, 1
+; GFX9-NEXT: s_sub_i32 s3, s1, s7
+; GFX9-NEXT: s_cmp_ge_u32 s1, s7
+; GFX9-NEXT: s_cselect_b32 s0, s2, s0
+; GFX9-NEXT: s_cselect_b32 s1, s3, s1
+; GFX9-NEXT: s_add_i32 s2, s0, 1
+; GFX9-NEXT: s_cmp_ge_u32 s1, s7
+; GFX9-NEXT: s_cselect_b32 s0, s2, s0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-NEXT: s_endpgm
%r = udiv i32 %x, %y
store i32 %r, ptr addrspace(1) %out
@@ -167,29 +167,29 @@ define amdgpu_kernel void @urem_i32(ptr addrspace(1) %out, i32 %x, i32 %y) {
;
; GFX9-LABEL: urem_i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3
-; GFX9-NEXT: s_sub_i32 s4, 0, s3
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7
+; GFX9-NEXT: s_sub_i32 s0, 0, s7
; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: v_readfirstlane_b32 s5, v0
-; GFX9-NEXT: s_mul_i32 s4, s4, s5
-; GFX9-NEXT: s_mul_hi_u32 s4, s5, s4
-; GFX9-NEXT: s_add_i32 s5, s5, s4
-; GFX9-NEXT: s_mul_hi_u32 s4, s2, s5
-; GFX9-NEXT: s_mul_i32 s4, s4, s3
-; GFX9-NEXT: s_sub_i32 s2, s2, s4
-; GFX9-NEXT: s_sub_i32 s4, s2, s3
-; GFX9-NEXT: s_cmp_ge_u32 s2, s3
-; GFX9-NEXT: s_cselect_b32 s2, s4, s2
-; GFX9-NEXT: s_sub_i32 s4, s2, s3
-; GFX9-NEXT: s_cmp_ge_u32 s2, s3
-; GFX9-NEXT: s_cselect_b32 s2, s4, s2
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_mul_i32 s0, s0, s1
+; GFX9-NEXT: s_mul_hi_u32 s0, s1, s0
+; GFX9-NEXT: s_add_i32 s1, s1, s0
+; GFX9-NEXT: s_mul_hi_u32 s0, s6, s1
+; GFX9-NEXT: s_mul_i32 s0, s0, s7
+; GFX9-NEXT: s_sub_i32 s0, s6, s0
+; GFX9-NEXT: s_sub_i32 s1, s0, s7
+; GFX9-NEXT: s_cmp_ge_u32 s0, s7
+; GFX9-NEXT: s_cselect_b32 s0, s1, s0
+; GFX9-NEXT: s_sub_i32 s1, s0, s7
+; GFX9-NEXT: s_cmp_ge_u32 s0, s7
+; GFX9-NEXT: s_cselect_b32 s0, s1, s0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-NEXT: s_endpgm
%r = urem i32 %x, %y
store i32 %r, ptr addrspace(1) %out
@@ -280,37 +280,37 @@ define amdgpu_kernel void @sdiv_i32(ptr addrspace(1) %out, i32 %x, i32 %y) {
;
; GFX9-LABEL: sdiv_i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_abs_i32 s4, s3
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s4
-; GFX9-NEXT: s_sub_i32 s5, 0, s4
-; GFX9-NEXT: s_xor_b32 s3, s2, s3
-; GFX9-NEXT: s_abs_i32 s2, s2
+; GFX9-NEXT: s_abs_i32 s0, s7
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s0
+; GFX9-NEXT: s_xor_b32 s1, s6, s7
+; GFX9-NEXT: s_abs_i32 s2, s6
+; GFX9-NEXT: s_sub_i32 s3, 0, s0
; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT: s_ashr_i32 s3, s3, 31
+; GFX9-NEXT: s_ashr_i32 s1, s1, 31
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_readfirstlane_b32 s6, v0
-; GFX9-NEXT: s_mul_i32 s5, s5, s6
-; GFX9-NEXT: s_mul_hi_u32 s5, s6, s5
-; GFX9-NEXT: s_add_i32 s6, s6, s5
-; GFX9-NEXT: s_mul_hi_u32 s5, s2, s6
-; GFX9-NEXT: s_mul_i32 s6, s5, s4
+; GFX9-NEXT: s_mul_i32 s3, s3, s6
+; GFX9-NEXT: s_mul_hi_u32 s3, s6, s3
+; GFX9-NEXT: s_add_i32 s6, s6, s3
+; GFX9-NEXT: s_mul_hi_u32 s3, s2, s6
+; GFX9-NEXT: s_mul_i32 s6, s3, s0
; GFX9-NEXT: s_sub_i32 s2, s2, s6
-; GFX9-NEXT: s_add_i32 s7, s5, 1
-; GFX9-NEXT: s_sub_i32 s6, s2, s4
-; GFX9-NEXT: s_cmp_ge_u32 s2, s4
-; GFX9-NEXT: s_cselect_b32 s5, s7, s5
+; GFX9-NEXT: s_add_i32 s7, s3, 1
+; GFX9-NEXT: s_sub_i32 s6, s2, s0
+; GFX9-NEXT: s_cmp_ge_u32 s2, s0
+; GFX9-NEXT: s_cselect_b32 s3, s7, s3
; GFX9-NEXT: s_cselect_b32 s2, s6, s2
-; GFX9-NEXT: s_add_i32 s6, s5, 1
-; GFX9-NEXT: s_cmp_ge_u32 s2, s4
-; GFX9-NEXT: s_cselect_b32 s2, s6, s5
-; GFX9-NEXT: s_xor_b32 s2, s2, s3
-; GFX9-NEXT: s_sub_i32 s2, s2, s3
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT: s_add_i32 s6, s3, 1
+; GFX9-NEXT: s_cmp_ge_u32 s2, s0
+; GFX9-NEXT: s_cselect_b32 s0, s6, s3
+; GFX9-NEXT: s_xor_b32 s0, s0, s1
+; GFX9-NEXT: s_sub_i32 s0, s0, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-NEXT: s_endpgm
%r = sdiv i32 %x, %y
store i32 %r, ptr addrspace(1) %out
@@ -394,34 +394,34 @@ define amdgpu_kernel void @srem_i32(ptr addrspace(1) %out, i32 %x, i32 %y) {
;
; GFX9-LABEL: srem_i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_abs_i32 s3, s3
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3
-; GFX9-NEXT: s_sub_i32 s5, 0, s3
-; GFX9-NEXT: s_ashr_i32 s4, s2, 31
-; GFX9-NEXT: s_abs_i32 s2, s2
+; GFX9-NEXT: s_abs_i32 s0, s7
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s0
+; GFX9-NEXT: s_ashr_i32 s1, s6, 31
+; GFX9-NEXT: s_abs_i32 s2, s6
+; GFX9-NEXT: s_sub_i32 s3, 0, s0
; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_readfirstlane_b32 s6, v0
-; GFX9-NEXT: s_mul_i32 s5, s5, s6
-; GFX9-NEXT: s_mul_hi_u32 s5, s6, s5
-; GFX9-NEXT: s_add_i32 s6, s6, s5
-; GFX9-NEXT: s_mul_hi_u32 s5, s2, s6
-; GFX9-NEXT: s_mul_i32 s5, s5, s3
-; GFX9-NEXT: s_sub_i32 s2, s2, s5
-; GFX9-NEXT: s_sub_i32 s5, s2, s3
-; GFX9-NEXT: s_cmp_ge_u32 s2, s3
-; GFX9-NEXT: s_cselect_b32 s2, s5, s2
-; GFX9-NEXT: s_sub_i32 s5, s2, s3
-; GFX9-NEXT: s_cmp_ge_u32 s2, s3
-; GFX9-NEXT: s_cselect_b32 s2, s5, s2
-; GFX9-NEXT: s_xor_b32 s2, s2, s4
-; GFX9-NEXT: s_sub_i32 s2, s2, s4
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT: s_mul_i32 s3, s3, s6
+; GFX9-NEXT: s_mul_hi_u32 s3, s6, s3
+; GFX9-NEXT: s_add_i32 s6, s6, s3
+; GFX9-NEXT: s_mul_hi_u32 s3, s2, s6
+; GFX9-NEXT: s_mul_i32 s3, s3, s0
+; GFX9-NEXT: s_sub_i32 s2, s2, s3
+; GFX9-NEXT: s_sub_i32 s3, s2, s0
+; GFX9-NEXT: s_cmp_ge_u32 s2, s0
+; GFX9-NEXT: s_cselect_b32 s2, s3, s2
+; GFX9-NEXT: s_sub_i32 s3, s2, s0
+; GFX9-NEXT: s_cmp_ge_u32 s2, s0
+; GFX9-NEXT: s_cselect_b32 s0, s3, s2
+; GFX9-NEXT: s_xor_b32 s0, s0, s1
+; GFX9-NEXT: s_sub_i32 s0, s0, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-NEXT: s_endpgm
%r = srem i32 %x, %y
store i32 %r, ptr addrspace(1) %out
@@ -5486,13 +5486,13 @@ define amdgpu_kernel void @udiv_i32_pow2_shl_denom(ptr addrspace(1) %out, i32 %x
;
; GFX9-LABEL: udiv_i32_pow2_shl_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_add_i32 s3, s3, 12
-; GFX9-NEXT: s_lshr_b32 s2, s2, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_add_i32 s0, s7, 12
+; GFX9-NEXT: s_lshr_b32 s0, s6, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: s_endpgm
%shl.y = shl i32 4096, %y
%r = udiv i32 %x, %shl.y
@@ -5528,14 +5528,14 @@ define amdgpu_kernel void @udiv_v2i32_pow2k_denom(ptr addrspace(1) %out, <2 x i3
;
; GFX9-LABEL: udiv_v2i32_pow2k_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s2, s2, 12
-; GFX9-NEXT: s_lshr_b32 s3, s3, 12
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT: s_lshr_b32 s0, s6, 12
+; GFX9-NEXT: s_lshr_b32 s1, s7, 12
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX9-NEXT: s_endpgm
%r = udiv <2 x i32> %x, <i32 4096, i32 4096>
store <2 x i32> %r, ptr addrspace(1) %out
@@ -5574,18 +5574,18 @@ define amdgpu_kernel void @udiv_v2i32_mixed_pow2k_denom(ptr addrspace(1) %out, <
;
; GFX9-LABEL: udiv_v2i32_mixed_pow2k_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_hi_u32 s4, s3, 0x100101
-; GFX9-NEXT: s_sub_i32 s3, s3, s4
-; GFX9-NEXT: s_lshr_b32 s3, s3, 1
-; GFX9-NEXT: s_add_i32 s3, s3, s4
-; GFX9-NEXT: s_lshr_b32 s2, s2, 12
-; GFX9-NEXT: s_lshr_b32 s3, s3, 11
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT: s_mul_hi_u32 s1, s7, 0x100101
+; GFX9-NEXT: s_sub_i32 s2, s7, s1
+; GFX9-NEXT: s_lshr_b32 s2, s2, 1
+; GFX9-NEXT: s_add_i32 s2, s2, s1
+; GFX9-NEXT: s_lshr_b32 s0, s6, 12
+; GFX9-NEXT: s_lshr_b32 s1, s2, 11
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX9-NEXT: s_endpgm
%r = udiv <2 x i32> %x, <i32 4096, i32 4095>
store <2 x i32> %r, ptr addrspace(1) %out
@@ -5879,14 +5879,14 @@ define amdgpu_kernel void @urem_i32_pow2_shl_denom(ptr addrspace(1) %out, i32 %x
;
; GFX9-LABEL: urem_i32_pow2_shl_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b32 s3, 0x1000, s3
-; GFX9-NEXT: s_add_i32 s3, s3, -1
-; GFX9-NEXT: s_and_b32 s2, s2, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_lshl_b32 s0, 0x1000, s7
+; GFX9-NEXT: s_add_i32 s0, s0, -1
+; GFX9-NEXT: s_and_b32 s0, s6, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: s_endpgm
%shl.y = shl i32 4096, %y
%r = urem i32 %x, %shl.y
@@ -5922,14 +5922,14 @@ define amdgpu_kernel void @urem_v2i32_pow2k_denom(ptr addrspace(1) %out, <2 x i3
;
; GFX9-LABEL: urem_v2i32_pow2k_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s2, s2, 0xfff
-; GFX9-NEXT: s_and_b32 s3, s3, 0xfff
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT: s_and_b32 s0, s6, 0xfff
+; GFX9-NEXT: s_and_b32 s1, s7, 0xfff
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX9-NEXT: s_endpgm
%r = urem <2 x i32> %x, <i32 4096, i32 4096>
store <2 x i32> %r, ptr addrspace(1) %out
@@ -6239,41 +6239,41 @@ define amdgpu_kernel void @sdiv_i32_pow2_shl_denom(ptr addrspace(1) %out, i32 %x
;
; GFX9-LABEL: sdiv_i32_pow2_shl_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b32 s3, 0x1000, s3
-; GFX9-NEXT: s_ashr_i32 s4, s3, 31
-; GFX9-NEXT: s_add_i32 s3, s3, s4
-; GFX9-NEXT: s_xor_b32 s3, s3, s4
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3
-; GFX9-NEXT: s_sub_i32 s6, 0, s3
-; GFX9-NEXT: s_ashr_i32 s5, s2, 31
-; GFX9-NEXT: s_add_i32 s2, s2, s5
+; GFX9-NEXT: s_lshl_b32 s0, 0x1000, s7
+; GFX9-NEXT: s_ashr_i32 s1, s0, 31
+; GFX9-NEXT: s_add_i32 s0, s0, s1
+; GFX9-NEXT: s_xor_b32 s0, s0, s1
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s0
+; GFX9-NEXT: s_ashr_i32 s2, s6, 31
+; GFX9-NEXT: s_add_i32 s3, s6, s2
+; GFX9-NEXT: s_sub_i32 s6, 0, s0
; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT: s_xor_b32 s2, s2, s5
+; GFX9-NEXT: s_xor_b32 s3, s3, s2
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_readfirstlane_b32 s7, v0
; GFX9-NEXT: s_mul_i32 s6, s6, s7
; GFX9-NEXT: s_mul_hi_u32 s6, s7, s6
; GFX9-NEXT: s_add_i32 s7, s7, s6
-; GFX9-NEXT: s_mul_hi_u32 s6, s2, s7
-; GFX9-NEXT: s_mul_i32 s8, s6, s3
-; GFX9-NEXT: s_sub_i32 s2, s2, s8
+; GFX9-NEXT: s_mul_hi_u32 s6, s3, s7
+; GFX9-NEXT: s_mul_i32 s8, s6, s0
+; GFX9-NEXT: s_sub_i32 s3, s3, s8
; GFX9-NEXT: s_add_i32 s7, s6, 1
-; GFX9-NEXT: s_sub_i32 s8, s2, s3
-; GFX9-NEXT: s_cmp_ge_u32 s2, s3
+; GFX9-NEXT: s_sub_i32 s8, s3, s0
+; GFX9-NEXT: s_cmp_ge_u32 s3, s0
; GFX9-NEXT: s_cselect_b32 s6, s7, s6
-; GFX9-NEXT: s_cselect_b32 s2, s8, s2
+; GFX9-NEXT: s_cselect_b32 s3, s8, s3
; GFX9-NEXT: s_add_i32 s7, s6, 1
-; GFX9-NEXT: s_cmp_ge_u32 s2, s3
-; GFX9-NEXT: s_cselect_b32 s2, s7, s6
-; GFX9-NEXT: s_xor_b32 s3, s5, s4
-; GFX9-NEXT: s_xor_b32 s2, s2, s3
-; GFX9-NEXT: s_sub_i32 s2, s2, s3
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT: s_cmp_ge_u32 s3, s0
+; GFX9-NEXT: s_cselect_b32 s0, s7, s6
+; GFX9-NEXT: s_xor_b32 s1, s2, s1
+; GFX9-NEXT: s_xor_b32 s0, s0, s1
+; GFX9-NEXT: s_sub_i32 s0, s0, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-NEXT: s_endpgm
%shl.y = shl i32 4096, %y
%r = sdiv i32 %x, %shl.y
@@ -6315,20 +6315,20 @@ define amdgpu_kernel void @sdiv_v2i32_pow2k_denom(ptr addrspace(1) %out, <2 x i3
;
; GFX9-LABEL: sdiv_v2i32_pow2k_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_ashr_i32 s4, s2, 31
-; GFX9-NEXT: s_ashr_i32 s5, s3, 31
-; GFX9-NEXT: s_lshr_b32 s4, s4, 20
-; GFX9-NEXT: s_lshr_b32 s5, s5, 20
-; GFX9-NEXT: s_add_i32 s2, s2, s4
-; GFX9-NEXT: s_add_i32 s3, s3, s5
-; GFX9-NEXT: s_ashr_i32 s2, s2, 12
-; GFX9-NEXT: s_ashr_i32 s3, s3, 12
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT: s_ashr_i32 s0, s6, 31
+; GFX9-NEXT: s_ashr_i32 s1, s7, 31
+; GFX9-NEXT: s_lshr_b32 s0, s0, 20
+; GFX9-NEXT: s_lshr_b32 s1, s1, 20
+; GFX9-NEXT: s_add_i32 s0, s6, s0
+; GFX9-NEXT: s_add_i32 s1, s7, s1
+; GFX9-NEXT: s_ashr_i32 s0, s0, 12
+; GFX9-NEXT: s_ashr_i32 s1, s1, 12
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX9-NEXT: s_endpgm
%r = sdiv <2 x i32> %x, <i32 4096, i32 4096>
store <2 x i32> %r, ptr addrspace(1) %out
@@ -6370,21 +6370,21 @@ define amdgpu_kernel void @ssdiv_v2i32_mixed_pow2k_denom(ptr addrspace(1) %out,
;
; GFX9-LABEL: ssdiv_v2i32_mixed_pow2k_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_ashr_i32 s4, s2, 31
-; GFX9-NEXT: s_mul_hi_i32 s5, s3, 0x80080081
-; GFX9-NEXT: s_lshr_b32 s4, s4, 20
-; GFX9-NEXT: s_add_i32 s5, s5, s3
-; GFX9-NEXT: s_add_i32 s2, s2, s4
-; GFX9-NEXT: s_lshr_b32 s3, s5, 31
-; GFX9-NEXT: s_ashr_i32 s4, s5, 11
-; GFX9-NEXT: s_ashr_i32 s2, s2, 12
-; GFX9-NEXT: s_add_i32 s4, s4, s3
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT: s_ashr_i32 s0, s6, 31
+; GFX9-NEXT: s_mul_hi_i32 s1, s7, 0x80080081
+; GFX9-NEXT: s_lshr_b32 s0, s0, 20
+; GFX9-NEXT: s_add_i32 s1, s1, s7
+; GFX9-NEXT: s_add_i32 s0, s6, s0
+; GFX9-NEXT: s_lshr_b32 s2, s1, 31
+; GFX9-NEXT: s_ashr_i32 s1, s1, 11
+; GFX9-NEXT: s_ashr_i32 s0, s0, 12
+; GFX9-NEXT: s_add_i32 s1, s1, s2
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX9-NEXT: s_endpgm
%r = sdiv <2 x i32> %x, <i32 4096, i32 4095>
store <2 x i32> %r, ptr addrspace(1) %out
@@ -6753,38 +6753,38 @@ define amdgpu_kernel void @srem_i32_pow2_shl_denom(ptr addrspace(1) %out, i32 %x
;
; GFX9-LABEL: srem_i32_pow2_shl_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b32 s3, 0x1000, s3
-; GFX9-NEXT: s_ashr_i32 s4, s3, 31
-; GFX9-NEXT: s_add_i32 s3, s3, s4
-; GFX9-NEXT: s_xor_b32 s3, s3, s4
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3
-; GFX9-NEXT: s_sub_i32 s5, 0, s3
-; GFX9-NEXT: s_ashr_i32 s4, s2, 31
-; GFX9-NEXT: s_add_i32 s2, s2, s4
+; GFX9-NEXT: s_lshl_b32 s0, 0x1000, s7
+; GFX9-NEXT: s_ashr_i32 s1, s0, 31
+; GFX9-NEXT: s_add_i32 s0, s0, s1
+; GFX9-NEXT: s_xor_b32 s0, s0, s1
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s0
+; GFX9-NEXT: s_ashr_i32 s1, s6, 31
+; GFX9-NEXT: s_add_i32 s2, s6, s1
+; GFX9-NEXT: s_sub_i32 s3, 0, s0
; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT: s_xor_b32 s2, s2, s4
+; GFX9-NEXT: s_xor_b32 s2, s2, s1
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_readfirstlane_b32 s6, v0
-; GFX9-NEXT: s_mul_i32 s5, s5, s6
-; GFX9-NEXT: s_mul_hi_u32 s5, s6, s5
-; GFX9-NEXT: s_add_i32 s6, s6, s5
-; GFX9-NEXT: s_mul_hi_u32 s5, s2, s6
-; GFX9-NEXT: s_mul_i32 s5, s5, s3
-; GFX9-NEXT: s_sub_i32 s2, s2, s5
-; GFX9-NEXT: s_sub_i32 s5, s2, s3
-; GFX9-NEXT: s_cmp_ge_u32 s2, s3
-; GFX9-NEXT: s_cselect_b32 s2, s5, s2
-; GFX9-NEXT: s_sub_i32 s5, s2, s3
-; GFX9-NEXT: s_cmp_ge_u32 s2, s3
-; GFX9-NEXT: s_cselect_b32 s2, s5, s2
-; GFX9-NEXT: s_xor_b32 s2, s2, s4
-; GFX9-NEXT: s_sub_i32 s2, s2, s4
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT: s_mul_i32 s3, s3, s6
+; GFX9-NEXT: s_mul_hi_u32 s3, s6, s3
+; GFX9-NEXT: s_add_i32 s6, s6, s3
+; GFX9-NEXT: s_mul_hi_u32 s3, s2, s6
+; GFX9-NEXT: s_mul_i32 s3, s3, s0
+; GFX9-NEXT: s_sub_i32 s2, s2, s3
+; GFX9-NEXT: s_sub_i32 s3, s2, s0
+; GFX9-NEXT: s_cmp_ge_u32 s2, s0
+; GFX9-NEXT: s_cselect_b32 s2, s3, s2
+; GFX9-NEXT: s_sub_i32 s3, s2, s0
+; GFX9-NEXT: s_cmp_ge_u32 s2, s0
+; GFX9-NEXT: s_cselect_b32 s0, s3, s2
+; GFX9-NEXT: s_xor_b32 s0, s0, s1
+; GFX9-NEXT: s_sub_i32 s0, s0, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-NEXT: s_endpgm
%shl.y = shl i32 4096, %y
%r = srem i32 %x, %shl.y
@@ -6828,22 +6828,22 @@ define amdgpu_kernel void @srem_v2i32_pow2k_denom(ptr addrspace(1) %out, <2 x i3
;
; GFX9-LABEL: srem_v2i32_pow2k_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_ashr_i32 s4, s2, 31
-; GFX9-NEXT: s_ashr_i32 s5, s3, 31
-; GFX9-NEXT: s_lshr_b32 s4, s4, 20
-; GFX9-NEXT: s_lshr_b32 s5, s5, 20
-; GFX9-NEXT: s_add_i32 s4, s2, s4
-; GFX9-NEXT: s_add_i32 s5, s3, s5
-; GFX9-NEXT: s_and_b32 s4, s4, 0xfffff000
-; GFX9-NEXT: s_sub_i32 s2, s2, s4
-; GFX9-NEXT: s_and_b32 s4, s5, 0xfffff000
-; GFX9-NEXT: s_sub_i32 s3, s3, s4
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT: s_ashr_i32 s0, s6, 31
+; GFX9-NEXT: s_ashr_i32 s1, s7, 31
+; GFX9-NEXT: s_lshr_b32 s0, s0, 20
+; GFX9-NEXT: s_lshr_b32 s1, s1, 20
+; GFX9-NEXT: s_add_i32 s0, s6, s0
+; GFX9-NEXT: s_add_i32 s1, s7, s1
+; GFX9-NEXT: s_and_b32 s0, s0, 0xfffff000
+; GFX9-NEXT: s_and_b32 s1, s1, 0xfffff000
+; GFX9-NEXT: s_sub_i32 s0, s6, s0
+; GFX9-NEXT: s_sub_i32 s1, s7, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX9-NEXT: s_endpgm
%r = srem <2 x i32> %x, <i32 4096, i32 4096>
store <2 x i32> %r, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/build_vector.ll b/llvm/test/CodeGen/AMDGPU/build_vector.ll
index 8293280609517..13c4ff8b2ff30 100644
--- a/llvm/test/CodeGen/AMDGPU/build_vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/build_vector.ll
@@ -316,14 +316,14 @@ define amdgpu_kernel void @build_v2i32_from_v4i16_shuffle(ptr addrspace(1) %out,
;
; GFX940-LABEL: build_v2i32_from_v4i16_shuffle:
; GFX940: ; %bb.0: ; %entry
-; GFX940-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX940-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX940-NEXT: v_mov_b32_e32 v2, 0
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
-; GFX940-NEXT: s_lshl_b32 s3, s3, 16
-; GFX940-NEXT: s_lshl_b32 s2, s2, 16
-; GFX940-NEXT: v_mov_b32_e32 v0, s2
-; GFX940-NEXT: v_mov_b32_e32 v1, s3
-; GFX940-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
+; GFX940-NEXT: s_lshl_b32 s0, s7, 16
+; GFX940-NEXT: s_lshl_b32 s1, s6, 16
+; GFX940-NEXT: v_mov_b32_e32 v0, s1
+; GFX940-NEXT: v_mov_b32_e32 v1, s0
+; GFX940-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5] sc0 sc1
; GFX940-NEXT: s_endpgm
entry:
%shuf = shufflevector <4 x i16> %in, <4 x i16> zeroinitializer, <2 x i32> <i32 0, i32 2>
diff --git a/llvm/test/CodeGen/AMDGPU/cluster_stores.ll b/llvm/test/CodeGen/AMDGPU/cluster_stores.ll
index fad1d47f55fd7..bada3d904fbe3 100644
--- a/llvm/test/CodeGen/AMDGPU/cluster_stores.ll
+++ b/llvm/test/CodeGen/AMDGPU/cluster_stores.ll
@@ -49,20 +49,20 @@ define amdgpu_kernel void @cluster_load_cluster_store(ptr noalias %lb, ptr noali
;
; GFX10-LABEL: cluster_load_cluster_store:
; GFX10: ; %bb.0: ; %bb
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_add_u32 s4, s0, 8
-; GFX10-NEXT: s_addc_u32 s5, s1, 0
-; GFX10-NEXT: s_add_u32 s6, s0, 16
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: s_addc_u32 s7, s1, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: s_add_u32 s0, s0, 24
-; GFX10-NEXT: v_mov_b32_e32 v2, s4
-; GFX10-NEXT: s_addc_u32 s1, s1, 0
-; GFX10-NEXT: v_mov_b32_e32 v3, s5
-; GFX10-NEXT: v_mov_b32_e32 v4, s6
-; GFX10-NEXT: v_mov_b32_e32 v5, s7
+; GFX10-NEXT: s_add_u32 s0, s4, 8
+; GFX10-NEXT: s_addc_u32 s1, s5, 0
+; GFX10-NEXT: s_add_u32 s2, s4, 16
+; GFX10-NEXT: v_mov_b32_e32 v3, s1
+; GFX10-NEXT: s_addc_u32 s3, s5, 0
+; GFX10-NEXT: v_mov_b32_e32 v0, s4
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GFX10-NEXT: s_add_u32 s0, s4, 24
+; GFX10-NEXT: v_mov_b32_e32 v1, s5
+; GFX10-NEXT: s_addc_u32 s1, s5, 0
+; GFX10-NEXT: v_mov_b32_e32 v5, s3
+; GFX10-NEXT: v_mov_b32_e32 v4, s2
; GFX10-NEXT: v_mov_b32_e32 v7, s1
; GFX10-NEXT: v_mov_b32_e32 v6, s0
; GFX10-NEXT: s_clause 0x3
@@ -70,16 +70,16 @@ define amdgpu_kernel void @cluster_load_cluster_store(ptr noalias %lb, ptr noali
; GFX10-NEXT: flat_load_dword v9, v[2:3]
; GFX10-NEXT: flat_load_dword v10, v[4:5]
; GFX10-NEXT: flat_load_dword v11, v[6:7]
-; GFX10-NEXT: s_add_u32 s0, s2, 8
-; GFX10-NEXT: s_addc_u32 s1, s3, 0
-; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: s_add_u32 s0, s6, 8
+; GFX10-NEXT: s_addc_u32 s1, s7, 0
+; GFX10-NEXT: v_mov_b32_e32 v0, s6
; GFX10-NEXT: v_mov_b32_e32 v3, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: s_add_u32 s0, s2, 16
-; GFX10-NEXT: s_addc_u32 s1, s3, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-NEXT: s_add_u32 s2, s2, 24
-; GFX10-NEXT: s_addc_u32 s3, s3, 0
+; GFX10-NEXT: s_add_u32 s0, s6, 16
+; GFX10-NEXT: s_addc_u32 s1, s7, 0
+; GFX10-NEXT: s_add_u32 s2, s6, 24
+; GFX10-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-NEXT: s_addc_u32 s3, s7, 0
; GFX10-NEXT: v_mov_b32_e32 v5, s1
; GFX10-NEXT: v_mov_b32_e32 v4, s0
; GFX10-NEXT: v_mov_b32_e32 v7, s3
@@ -175,20 +175,20 @@ define amdgpu_kernel void @cluster_load_valu_cluster_store(ptr noalias %lb, ptr
;
; GFX10-LABEL: cluster_load_valu_cluster_store:
; GFX10: ; %bb.0: ; %bb
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_add_u32 s4, s0, 8
-; GFX10-NEXT: s_addc_u32 s5, s1, 0
-; GFX10-NEXT: v_mov_b32_e32 v2, s4
-; GFX10-NEXT: s_add_u32 s6, s0, 16
-; GFX10-NEXT: v_mov_b32_e32 v3, s5
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: s_addc_u32 s7, s1, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: s_add_u32 s0, s0, 24
-; GFX10-NEXT: s_addc_u32 s1, s1, 0
-; GFX10-NEXT: v_mov_b32_e32 v4, s6
-; GFX10-NEXT: v_mov_b32_e32 v5, s7
+; GFX10-NEXT: s_add_u32 s0, s4, 8
+; GFX10-NEXT: s_addc_u32 s1, s5, 0
+; GFX10-NEXT: s_add_u32 s2, s4, 16
+; GFX10-NEXT: v_mov_b32_e32 v3, s1
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GFX10-NEXT: s_addc_u32 s3, s5, 0
+; GFX10-NEXT: s_add_u32 s0, s4, 24
+; GFX10-NEXT: v_mov_b32_e32 v0, s4
+; GFX10-NEXT: s_addc_u32 s1, s5, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, s5
+; GFX10-NEXT: v_mov_b32_e32 v5, s3
+; GFX10-NEXT: v_mov_b32_e32 v4, s2
; GFX10-NEXT: flat_load_dword v6, v[2:3]
; GFX10-NEXT: v_mov_b32_e32 v3, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s0
@@ -196,18 +196,18 @@ define amdgpu_kernel void @cluster_load_valu_cluster_store(ptr noalias %lb, ptr
; GFX10-NEXT: flat_load_dword v8, v[0:1]
; GFX10-NEXT: flat_load_dword v9, v[4:5]
; GFX10-NEXT: flat_load_dword v10, v[2:3]
-; GFX10-NEXT: s_add_u32 s0, s2, 8
-; GFX10-NEXT: s_addc_u32 s1, s3, 0
-; GFX10-NEXT: s_add_u32 s4, s2, 16
+; GFX10-NEXT: s_add_u32 s0, s6, 8
+; GFX10-NEXT: s_addc_u32 s1, s7, 0
+; GFX10-NEXT: s_add_u32 s2, s6, 16
; GFX10-NEXT: v_mov_b32_e32 v3, s1
-; GFX10-NEXT: s_addc_u32 s5, s3, 0
-; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: s_addc_u32 s3, s7, 0
+; GFX10-NEXT: v_mov_b32_e32 v0, s6
; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: s_add_u32 s0, s2, 24
-; GFX10-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-NEXT: v_mov_b32_e32 v4, s4
-; GFX10-NEXT: s_addc_u32 s1, s3, 0
-; GFX10-NEXT: v_mov_b32_e32 v5, s5
+; GFX10-NEXT: s_add_u32 s0, s6, 24
+; GFX10-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-NEXT: v_mov_b32_e32 v5, s3
+; GFX10-NEXT: s_addc_u32 s1, s7, 0
+; GFX10-NEXT: v_mov_b32_e32 v4, s2
; GFX10-NEXT: s_waitcnt vmcnt(3) lgkmcnt(3)
; GFX10-NEXT: v_add_nc_u32_e32 v11, 1, v6
; GFX10-NEXT: v_mov_b32_e32 v7, s1
diff --git a/llvm/test/CodeGen/AMDGPU/combine-cond-add-sub.ll b/llvm/test/CodeGen/AMDGPU/combine-cond-add-sub.ll
index df223b3ec1354..b5e0589cf9e46 100644
--- a/llvm/test/CodeGen/AMDGPU/combine-cond-add-sub.ll
+++ b/llvm/test/CodeGen/AMDGPU/combine-cond-add-sub.ll
@@ -665,17 +665,17 @@ define amdgpu_kernel void @sub_zext_setcc_commute(ptr addrspace(1) nocapture %ar
;
; GFX9-LABEL: sub_zext_setcc_commute:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v3, v2, s[0:1]
+; GFX9-NEXT: global_load_dword v3, v2, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_sub_u32_e32 v0, v0, v3
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v0
-; GFX9-NEXT: v_subrev_u32_e32 v0, s3, v0
-; GFX9-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX9-NEXT: v_add_u32_e32 v0, s6, v0
+; GFX9-NEXT: v_subrev_u32_e32 v0, s7, v0
+; GFX9-NEXT: global_store_dword v2, v0, s[4:5]
; GFX9-NEXT: s_endpgm
bb:
%x = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -714,17 +714,17 @@ define amdgpu_kernel void @sub_sext_setcc_commute(ptr addrspace(1) nocapture %ar
;
; GFX9-LABEL: sub_sext_setcc_commute:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v3, v2, s[0:1]
+; GFX9-NEXT: global_load_dword v3, v2, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_sub_u32_e32 v0, v0, v3
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v0
-; GFX9-NEXT: v_subrev_u32_e32 v0, s3, v0
-; GFX9-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX9-NEXT: v_add_u32_e32 v0, s6, v0
+; GFX9-NEXT: v_subrev_u32_e32 v0, s7, v0
+; GFX9-NEXT: global_store_dword v2, v0, s[4:5]
; GFX9-NEXT: s_endpgm
bb:
%x = tail call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/ctlz.ll b/llvm/test/CodeGen/AMDGPU/ctlz.ll
index 63b9d68123fa4..ba0a1e75e29b7 100644
--- a/llvm/test/CodeGen/AMDGPU/ctlz.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctlz.ll
@@ -164,28 +164,28 @@ define amdgpu_kernel void @v_ctlz_i32(ptr addrspace(1) noalias %out, ptr addrspa
;
; GFX10-LABEL: v_ctlz_i32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbh_u32_e32 v0, v0
; GFX10-NEXT: v_min_u32_e32 v0, 32, v0
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_ctlz_i32:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-GISEL-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
; GFX10-GISEL-NEXT: v_min_u32_e32 v0, 32, v0
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-LABEL: v_ctlz_i32:
@@ -278,32 +278,32 @@ define amdgpu_kernel void @v_ctlz_v2i32(ptr addrspace(1) noalias %out, ptr addrs
;
; GFX10-LABEL: v_ctlz_v2i32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx2 v[0:1], v0, s[2:3]
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbh_u32_e32 v1, v1
; GFX10-NEXT: v_ffbh_u32_e32 v0, v0
; GFX10-NEXT: v_min_u32_e32 v1, 32, v1
; GFX10-NEXT: v_min_u32_e32 v0, 32, v0
-; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_ctlz_v2i32:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_dwordx2 v[0:1], v0, s[2:3]
+; GFX10-GISEL-NEXT: global_load_dwordx2 v[0:1], v0, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
; GFX10-GISEL-NEXT: v_min_u32_e32 v0, 32, v0
; GFX10-GISEL-NEXT: v_min_u32_e32 v1, 32, v1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-LABEL: v_ctlz_v2i32:
@@ -414,11 +414,11 @@ define amdgpu_kernel void @v_ctlz_v4i32(ptr addrspace(1) noalias %out, ptr addrs
;
; GFX10-LABEL: v_ctlz_v4i32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 4, v0
; GFX10-NEXT: v_mov_b32_e32 v4, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx4 v[0:3], v0, s[2:3]
+; GFX10-NEXT: global_load_dwordx4 v[0:3], v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbh_u32_e32 v3, v3
; GFX10-NEXT: v_ffbh_u32_e32 v2, v2
@@ -428,16 +428,16 @@ define amdgpu_kernel void @v_ctlz_v4i32(ptr addrspace(1) noalias %out, ptr addrs
; GFX10-NEXT: v_min_u32_e32 v2, 32, v2
; GFX10-NEXT: v_min_u32_e32 v1, 32, v1
; GFX10-NEXT: v_min_u32_e32 v0, 32, v0
-; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_ctlz_v4i32:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 4, v0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_dwordx4 v[0:3], v0, s[2:3]
+; GFX10-GISEL-NEXT: global_load_dwordx4 v[0:3], v0, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
@@ -447,7 +447,7 @@ define amdgpu_kernel void @v_ctlz_v4i32(ptr addrspace(1) noalias %out, ptr addrs
; GFX10-GISEL-NEXT: v_min_u32_e32 v1, 32, v1
; GFX10-GISEL-NEXT: v_min_u32_e32 v2, 32, v2
; GFX10-GISEL-NEXT: v_min_u32_e32 v3, 32, v3
-; GFX10-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX10-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-LABEL: v_ctlz_v4i32:
@@ -555,28 +555,28 @@ define amdgpu_kernel void @v_ctlz_i8(ptr addrspace(1) noalias %out, ptr addrspac
;
; GFX10-LABEL: v_ctlz_i8:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_ubyte v1, v0, s[2:3]
+; GFX10-NEXT: global_load_ubyte v1, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbh_u32_e32 v1, v1
; GFX10-NEXT: v_min_u32_e32 v1, 32, v1
; GFX10-NEXT: v_subrev_nc_u32_e32 v1, 24, v1
-; GFX10-NEXT: global_store_byte v0, v1, s[0:1]
+; GFX10-NEXT: global_store_byte v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_ctlz_i8:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_ubyte v1, v0, s[2:3]
+; GFX10-GISEL-NEXT: global_load_ubyte v1, v0, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
; GFX10-GISEL-NEXT: v_min_u32_e32 v1, 32, v1
; GFX10-GISEL-NEXT: v_subrev_nc_u32_e32 v1, 24, v1
-; GFX10-GISEL-NEXT: global_store_byte v0, v1, s[0:1]
+; GFX10-GISEL-NEXT: global_store_byte v0, v1, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-LABEL: v_ctlz_i8:
@@ -742,24 +742,24 @@ define amdgpu_kernel void @s_ctlz_i64_trunc(ptr addrspace(1) noalias %out, i64 %
;
; GFX10-LABEL: s_ctlz_i64_trunc:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_flbit_i32_b64 s2, s[2:3]
-; GFX10-NEXT: s_min_u32 s2, s2, 64
-; GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_flbit_i32_b64 s0, s[6:7]
+; GFX10-NEXT: s_min_u32 s0, s0, 64
+; GFX10-NEXT: v_mov_b32_e32 v1, s0
+; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: s_ctlz_i64_trunc:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: s_flbit_i32_b64 s2, s[2:3]
-; GFX10-GISEL-NEXT: s_min_u32 s2, s2, 64
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: s_flbit_i32_b64 s0, s[6:7]
+; GFX10-GISEL-NEXT: s_min_u32 s0, s0, 64
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-LABEL: s_ctlz_i64_trunc:
@@ -852,25 +852,25 @@ define amdgpu_kernel void @v_ctlz_i64(ptr addrspace(1) noalias %out, ptr addrspa
;
; GFX10-LABEL: v_ctlz_i64:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3]
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v2, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbh_u32_e32 v0, v0
; GFX10-NEXT: v_ffbh_u32_e32 v1, v1
; GFX10-NEXT: v_add_nc_u32_e64 v0, v0, 32 clamp
; GFX10-NEXT: v_min3_u32 v0, v0, v1, 64
; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_ctlz_i64:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3]
+; GFX10-GISEL-NEXT: global_load_dwordx2 v[0:1], v2, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
@@ -878,7 +878,7 @@ define amdgpu_kernel void @v_ctlz_i64(ptr addrspace(1) noalias %out, ptr addrspa
; GFX10-GISEL-NEXT: v_min_u32_e32 v0, v1, v0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: v_min_u32_e32 v0, 64, v0
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-LABEL: v_ctlz_i64:
@@ -981,33 +981,33 @@ define amdgpu_kernel void @v_ctlz_i64_trunc(ptr addrspace(1) noalias %out, ptr a
;
; GFX10-LABEL: v_ctlz_i64_trunc:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx2 v[1:2], v1, s[2:3]
+; GFX10-NEXT: global_load_dwordx2 v[1:2], v1, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbh_u32_e32 v1, v1
; GFX10-NEXT: v_ffbh_u32_e32 v2, v2
; GFX10-NEXT: v_add_nc_u32_e64 v1, v1, 32 clamp
; GFX10-NEXT: v_min3_u32 v1, v1, v2, 64
-; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_ctlz_i64_trunc:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_dwordx2 v[1:2], v1, s[2:3]
+; GFX10-GISEL-NEXT: global_load_dwordx2 v[1:2], v1, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v2, v2
; GFX10-GISEL-NEXT: v_add_nc_u32_e64 v1, v1, 32 clamp
; GFX10-GISEL-NEXT: v_min_u32_e32 v1, v2, v1
; GFX10-GISEL-NEXT: v_min_u32_e32 v1, 64, v1
-; GFX10-GISEL-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-GISEL-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-LABEL: v_ctlz_i64_trunc:
@@ -1099,29 +1099,29 @@ define amdgpu_kernel void @v_ctlz_i32_sel_eq_neg1(ptr addrspace(1) noalias %out,
;
; GFX10-LABEL: v_ctlz_i32_sel_eq_neg1:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbh_u32_e32 v0, v0
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_ctlz_i32_sel_eq_neg1:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-GISEL-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v1, v0
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-GISEL-NEXT: v_min_u32_e32 v1, 32, v1
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, v1, -1, vcc_lo
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-LABEL: v_ctlz_i32_sel_eq_neg1:
@@ -1208,29 +1208,29 @@ define amdgpu_kernel void @v_ctlz_i32_sel_ne_neg1(ptr addrspace(1) noalias %out,
;
; GFX10-LABEL: v_ctlz_i32_sel_ne_neg1:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbh_u32_e32 v0, v0
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_ctlz_i32_sel_ne_neg1:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-GISEL-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v1, v0
; GFX10-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GFX10-GISEL-NEXT: v_min_u32_e32 v1, 32, v1
; GFX10-GISEL-NEXT: v_cndmask_b32_e32 v0, -1, v1, vcc_lo
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-LABEL: v_ctlz_i32_sel_ne_neg1:
@@ -1326,32 +1326,32 @@ define amdgpu_kernel void @v_ctlz_i32_sel_eq_bitwidth(ptr addrspace(1) noalias %
;
; GFX10-LABEL: v_ctlz_i32_sel_eq_bitwidth:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbh_u32_e32 v0, v0
; GFX10-NEXT: v_min_u32_e32 v0, 32, v0
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 32, v0
; GFX10-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc_lo
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_ctlz_i32_sel_eq_bitwidth:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-GISEL-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
; GFX10-GISEL-NEXT: v_min_u32_e32 v0, 32, v0
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 32, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, v0, -1, vcc_lo
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-LABEL: v_ctlz_i32_sel_eq_bitwidth:
@@ -1450,32 +1450,32 @@ define amdgpu_kernel void @v_ctlz_i32_sel_ne_bitwidth(ptr addrspace(1) noalias %
;
; GFX10-LABEL: v_ctlz_i32_sel_ne_bitwidth:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbh_u32_e32 v0, v0
; GFX10-NEXT: v_min_u32_e32 v0, 32, v0
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 32, v0
; GFX10-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc_lo
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_ctlz_i32_sel_ne_bitwidth:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-GISEL-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
; GFX10-GISEL-NEXT: v_min_u32_e32 v0, 32, v0
; GFX10-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, 32, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc_lo
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-LABEL: v_ctlz_i32_sel_ne_bitwidth:
@@ -1569,22 +1569,22 @@ define amdgpu_kernel void @v_ctlz_i32_sel_ne_bitwidth(ptr addrspace(1) noalias %
;
; GFX10-LABEL: v_ctlz_i8_sel_eq_neg1:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_ubyte v0, v0, s[2:3]
+; GFX10-NEXT: global_load_ubyte v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbh_u32_e32 v0, v0
-; GFX10-NEXT: global_store_byte v1, v0, s[0:1]
+; GFX10-NEXT: global_store_byte v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_ctlz_i8_sel_eq_neg1:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s7
; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v2, v3, vcc_lo
; GFX10-GISEL-NEXT: global_load_ubyte v0, v[0:1], off
@@ -1595,7 +1595,7 @@ define amdgpu_kernel void @v_ctlz_i32_sel_ne_bitwidth(ptr addrspace(1) noalias %
; GFX10-GISEL-NEXT: v_subrev_nc_u32_e32 v1, 24, v1
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, v1, -1, vcc_lo
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: global_store_byte v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: global_store_byte v1, v0, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-LABEL: v_ctlz_i8_sel_eq_neg1:
@@ -1691,25 +1691,25 @@ define amdgpu_kernel void @v_ctlz_i32_sel_ne_bitwidth(ptr addrspace(1) noalias %
;
; GFX10-LABEL: v_ctlz_i16_sel_eq_neg1:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_ushort v1, v0, s[2:3]
+; GFX10-NEXT: global_load_ushort v1, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbh_u32_e32 v2, v1
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
; GFX10-NEXT: v_min_u32_e32 v2, 32, v2
; GFX10-NEXT: v_add_nc_u32_e32 v2, -16, v2
; GFX10-NEXT: v_cndmask_b32_e32 v1, 0xffff, v2, vcc_lo
-; GFX10-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_ctlz_i16_sel_eq_neg1:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_ushort v1, v0, s[2:3]
+; GFX10-GISEL-NEXT: global_load_ushort v1, v0, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbh_u32_e32 v2, v1
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
@@ -1717,7 +1717,7 @@ define amdgpu_kernel void @v_ctlz_i32_sel_ne_bitwidth(ptr addrspace(1) noalias %
; GFX10-GISEL-NEXT: v_subrev_nc_u32_e32 v2, 16, v2
; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, v2, 0xffff, vcc_lo
-; GFX10-GISEL-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-GISEL-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-LABEL: v_ctlz_i16_sel_eq_neg1:
@@ -1812,23 +1812,23 @@ define amdgpu_kernel void @v_ctlz_i7_sel_eq_neg1(ptr addrspace(1) noalias %out,
;
; GFX10-LABEL: v_ctlz_i7_sel_eq_neg1:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_ubyte v0, v0, s[2:3]
+; GFX10-NEXT: global_load_ubyte v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbh_u32_e32 v0, v0
; GFX10-NEXT: v_and_b32_e32 v0, 0x7f, v0
-; GFX10-NEXT: global_store_byte v1, v0, s[0:1]
+; GFX10-NEXT: global_store_byte v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_ctlz_i7_sel_eq_neg1:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s7
; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v2, v3, vcc_lo
; GFX10-GISEL-NEXT: global_load_ubyte v0, v[0:1], off
@@ -1841,7 +1841,7 @@ define amdgpu_kernel void @v_ctlz_i7_sel_eq_neg1(ptr addrspace(1) noalias %out,
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, v1, 0x7f, vcc_lo
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0x7f, v0
-; GFX10-GISEL-NEXT: global_store_byte v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: global_store_byte v1, v0, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-LABEL: v_ctlz_i7_sel_eq_neg1:
diff --git a/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll b/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
index f16f05811c185..a55c8cdc9b6e8 100644
--- a/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll
@@ -134,14 +134,14 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i32(ptr addrspace(1) noalias %out,
;
; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i32:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX9-GISEL-NEXT: global_load_dword v0, v0, s[6:7]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
-; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr i32, ptr addrspace(1) %valptr, i32 %tid
@@ -211,15 +211,15 @@ define amdgpu_kernel void @v_ctlz_zero_undef_v2i32(ptr addrspace(1) noalias %out
;
; GFX9-GISEL-LABEL: v_ctlz_zero_undef_v2i32:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dwordx2 v[0:1], v0, s[2:3]
+; GFX9-GISEL-NEXT: global_load_dwordx2 v[0:1], v0, s[6:7]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
-; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr <2 x i32>, ptr addrspace(1) %valptr, i32 %tid
@@ -295,17 +295,17 @@ define amdgpu_kernel void @v_ctlz_zero_undef_v4i32(ptr addrspace(1) noalias %out
;
; GFX9-GISEL-LABEL: v_ctlz_zero_undef_v4i32:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 4, v0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dwordx4 v[0:3], v0, s[2:3]
+; GFX9-GISEL-NEXT: global_load_dwordx4 v[0:3], v0, s[6:7]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v2
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v3, v3
-; GFX9-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX9-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr <4 x i32>, ptr addrspace(1) %valptr, i32 %tid
@@ -561,14 +561,14 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i64_with_select(ptr addrspace(1) no
;
; GFX9-GISEL-LABEL: s_ctlz_zero_undef_i64_with_select:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX9-GISEL-NEXT: s_mov_b32 s5, 0
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_mov_b32 s1, 0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_flbit_i32_b64 s4, s[2:3]
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_flbit_i32_b64 s0, s[6:7]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%ctlz = tail call i64 @llvm.ctlz.i64(i64 %val, i1 true) nounwind readnone
%ctlz_ret = icmp ne i64 %val, 0
@@ -649,17 +649,17 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i8_with_select(ptr addrspace(1) noa
;
; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i8_with_select:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[2:3]
+; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[6:7]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v1
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v2
; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
; GFX9-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, 32, v2, vcc
-; GFX9-GISEL-NEXT: global_store_byte v0, v1, s[0:1]
+; GFX9-GISEL-NEXT: global_store_byte v0, v1, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%val = load i8, ptr addrspace(1) %arrayidx, align 1
%ctlz = tail call i8 @llvm.ctlz.i8(i8 %val, i1 true) nounwind readnone
@@ -753,11 +753,11 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i16_with_select(ptr addrspace(1) no
;
; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i16_with_select:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-GISEL-NEXT: global_load_ubyte v2, v0, s[2:3] offset:1
+; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[6:7]
+; GFX9-GISEL-NEXT: global_load_ubyte v2, v0, s[6:7] offset:1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v2, 8, v1
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v1
@@ -765,7 +765,7 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i16_with_select(ptr addrspace(1) no
; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX9-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, 32, v2, vcc
-; GFX9-GISEL-NEXT: global_store_short v0, v1, s[0:1]
+; GFX9-GISEL-NEXT: global_store_short v0, v1, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%val = load i16, ptr addrspace(1) %arrayidx, align 1
%ctlz = tail call i16 @llvm.ctlz.i16(i16 %val, i1 true) nounwind readnone
@@ -869,13 +869,13 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i32_with_select(ptr addrspace(1) no
;
; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i32_with_select:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-GISEL-NEXT: global_load_ubyte v2, v0, s[2:3] offset:1
-; GFX9-GISEL-NEXT: global_load_ubyte v3, v0, s[2:3] offset:3
-; GFX9-GISEL-NEXT: global_load_ubyte v4, v0, s[2:3] offset:2
+; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[6:7]
+; GFX9-GISEL-NEXT: global_load_ubyte v2, v0, s[6:7] offset:1
+; GFX9-GISEL-NEXT: global_load_ubyte v3, v0, s[6:7] offset:3
+; GFX9-GISEL-NEXT: global_load_ubyte v4, v0, s[6:7] offset:2
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(2)
; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v2, 8, v1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(1)
@@ -886,7 +886,7 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i32_with_select(ptr addrspace(1) no
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v1
; GFX9-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, 32, v2, vcc
-; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%val = load i32, ptr addrspace(1) %arrayidx, align 1
%ctlz = tail call i32 @llvm.ctlz.i32(i32 %val, i1 true) nounwind readnone
@@ -1050,17 +1050,17 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i64_with_select(ptr addrspace(1) no
;
; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i64_with_select:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_ubyte v0, v1, s[2:3]
-; GFX9-GISEL-NEXT: global_load_ubyte v2, v1, s[2:3] offset:1
-; GFX9-GISEL-NEXT: global_load_ubyte v3, v1, s[2:3] offset:2
-; GFX9-GISEL-NEXT: global_load_ubyte v4, v1, s[2:3] offset:3
-; GFX9-GISEL-NEXT: global_load_ubyte v5, v1, s[2:3] offset:4
-; GFX9-GISEL-NEXT: global_load_ubyte v6, v1, s[2:3] offset:5
-; GFX9-GISEL-NEXT: global_load_ubyte v7, v1, s[2:3] offset:6
-; GFX9-GISEL-NEXT: global_load_ubyte v8, v1, s[2:3] offset:7
+; GFX9-GISEL-NEXT: global_load_ubyte v0, v1, s[6:7]
+; GFX9-GISEL-NEXT: global_load_ubyte v2, v1, s[6:7] offset:1
+; GFX9-GISEL-NEXT: global_load_ubyte v3, v1, s[6:7] offset:2
+; GFX9-GISEL-NEXT: global_load_ubyte v4, v1, s[6:7] offset:3
+; GFX9-GISEL-NEXT: global_load_ubyte v5, v1, s[6:7] offset:4
+; GFX9-GISEL-NEXT: global_load_ubyte v6, v1, s[6:7] offset:5
+; GFX9-GISEL-NEXT: global_load_ubyte v7, v1, s[6:7] offset:6
+; GFX9-GISEL-NEXT: global_load_ubyte v8, v1, s[6:7] offset:7
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(6)
; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v2, 8, v0
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(5)
@@ -1081,7 +1081,7 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i64_with_select(ptr addrspace(1) no
; GFX9-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX9-GISEL-NEXT: v_min_u32_e32 v0, v4, v0
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, 64, v0, vcc
-; GFX9-GISEL-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: global_store_dwordx2 v1, v[0:1], s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%val = load i64, ptr addrspace(1) %arrayidx, align 1
%ctlz = tail call i64 @llvm.ctlz.i64(i64 %val, i1 true) nounwind readnone
@@ -1158,11 +1158,11 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i8(ptr addrspace(1) noalias %out, p
;
; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i8:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s7
; GFX9-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v1, v0
; GFX9-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v3, vcc
; GFX9-GISEL-NEXT: global_load_ubyte v0, v[0:1], off
@@ -1170,7 +1170,7 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i8(ptr addrspace(1) noalias %out, p
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 24, v0
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
-; GFX9-GISEL-NEXT: global_store_byte v1, v0, s[0:1]
+; GFX9-GISEL-NEXT: global_store_byte v1, v0, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr i8, ptr addrspace(1) %valptr, i32 %tid
@@ -1282,12 +1282,12 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i64_trunc(ptr addrspace(1) noalias
;
; GFX9-GISEL-LABEL: s_ctlz_zero_undef_i64_trunc:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_flbit_i32_b64 s2, s[2:3]
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-GISEL-NEXT: s_flbit_i32_b64 s0, s[6:7]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%ctlz = call i64 @llvm.ctlz.i64(i64 %val, i1 true)
%trunc = trunc i64 %ctlz to i32
@@ -1364,17 +1364,17 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i64(ptr addrspace(1) noalias %out,
;
; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i64:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3]
+; GFX9-GISEL-NEXT: global_load_dwordx2 v[0:1], v2, s[6:7]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
; GFX9-GISEL-NEXT: v_add_u32_e32 v0, 32, v0
; GFX9-GISEL-NEXT: v_min_u32_e32 v0, v1, v0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr i64, ptr addrspace(1) %in, i32 %tid
@@ -1454,17 +1454,17 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i64_trunc(ptr addrspace(1) noalias
;
; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i64_trunc:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dwordx2 v[1:2], v1, s[2:3]
+; GFX9-GISEL-NEXT: global_load_dwordx2 v[1:2], v1, s[6:7]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v2
; GFX9-GISEL-NEXT: v_add_u32_e32 v1, 32, v1
; GFX9-GISEL-NEXT: v_min_u32_e32 v1, v2, v1
-; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr i64, ptr addrspace(1) %in, i32 %tid
@@ -1534,16 +1534,16 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i32_sel_eq_neg1(ptr addrspace(1) no
;
; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i32_sel_eq_neg1:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX9-GISEL-NEXT: global_load_dword v0, v0, s[6:7]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v0
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, v1, -1, vcc
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr i32, ptr addrspace(1) %valptr, i32 %tid
@@ -1613,16 +1613,16 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i32_sel_ne_neg1(ptr addrspace(1) no
;
; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i32_sel_ne_neg1:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX9-GISEL-NEXT: global_load_dword v0, v0, s[6:7]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v0
; GFX9-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, -1, v1, vcc
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr i32, ptr addrspace(1) %valptr, i32 %tid
@@ -1697,11 +1697,11 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i8_sel_eq_neg1(ptr addrspace(1) noa
;
; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i8_sel_eq_neg1:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s7
; GFX9-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v1, v0
; GFX9-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v3, vcc
; GFX9-GISEL-NEXT: global_load_ubyte v0, v[0:1], off
@@ -1709,9 +1709,9 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i8_sel_eq_neg1(ptr addrspace(1) noa
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v0
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v2
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_sdwa s[2:3], v0, v1 src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, v2, -1, s[2:3]
-; GFX9-GISEL-NEXT: global_store_byte v1, v0, s[0:1]
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_sdwa s[0:1], v0, v1 src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, v2, -1, s[0:1]
+; GFX9-GISEL-NEXT: global_store_byte v1, v0, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%valptr.gep = getelementptr i8, ptr addrspace(1) %valptr, i32 %tid
@@ -1799,17 +1799,17 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i32_sel_eq_neg1_two_use(ptr addrspa
;
; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i32_sel_eq_neg1_two_use:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX9-GISEL-NEXT: global_load_dword v0, v0, s[6:7]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v0
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, v2, -1, vcc
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: global_store_byte v[0:1], v2, off
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -1888,16 +1888,16 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i32_sel_eq_0(ptr addrspace(1) noali
;
; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i32_sel_eq_0:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX9-GISEL-NEXT: global_load_dword v0, v0, s[6:7]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v0
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, v1, 0, vcc
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr i32, ptr addrspace(1) %valptr, i32 %tid
@@ -1972,16 +1972,16 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i32_sel_ne_0(ptr addrspace(1) noali
;
; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i32_sel_ne_0:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX9-GISEL-NEXT: global_load_dword v0, v0, s[6:7]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v0
; GFX9-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr i32, ptr addrspace(1) %valptr, i32 %tid
@@ -2057,16 +2057,16 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i32_sel_eq_cmp_non0(ptr addrspace(1
;
; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i32_sel_eq_cmp_non0:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX9-GISEL-NEXT: global_load_dword v0, v0, s[6:7]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v0
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, v1, 0, vcc
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr i32, ptr addrspace(1) %valptr, i32 %tid
@@ -2142,16 +2142,16 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i32_sel_ne_cmp_non0(ptr addrspace(1
;
; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i32_sel_ne_cmp_non0:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX9-GISEL-NEXT: global_load_dword v0, v0, s[6:7]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v0
; GFX9-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 1, v0
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr i32, ptr addrspace(1) %valptr, i32 %tid
diff --git a/llvm/test/CodeGen/AMDGPU/cttz.ll b/llvm/test/CodeGen/AMDGPU/cttz.ll
index 02b0b1cc28fa8..57fe6cd4e1e45 100644
--- a/llvm/test/CodeGen/AMDGPU/cttz.ll
+++ b/llvm/test/CodeGen/AMDGPU/cttz.ll
@@ -148,28 +148,28 @@ define amdgpu_kernel void @v_cttz_i32(ptr addrspace(1) noalias %out, ptr addrspa
;
; GFX10-LABEL: v_cttz_i32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbl_b32_e32 v0, v0
; GFX10-NEXT: v_min_u32_e32 v0, 32, v0
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_cttz_i32:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-GISEL-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v0, v0
; GFX10-GISEL-NEXT: v_min_u32_e32 v0, 32, v0
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr i32, ptr addrspace(1) %valptr, i32 %tid
@@ -246,32 +246,32 @@ define amdgpu_kernel void @v_cttz_v2i32(ptr addrspace(1) noalias %out, ptr addrs
;
; GFX10-LABEL: v_cttz_v2i32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx2 v[0:1], v0, s[2:3]
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbl_b32_e32 v1, v1
; GFX10-NEXT: v_ffbl_b32_e32 v0, v0
; GFX10-NEXT: v_min_u32_e32 v1, 32, v1
; GFX10-NEXT: v_min_u32_e32 v0, 32, v0
-; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_cttz_v2i32:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_dwordx2 v[0:1], v0, s[2:3]
+; GFX10-GISEL-NEXT: global_load_dwordx2 v[0:1], v0, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v0, v0
; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v1, v1
; GFX10-GISEL-NEXT: v_min_u32_e32 v0, 32, v0
; GFX10-GISEL-NEXT: v_min_u32_e32 v1, 32, v1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr <2 x i32>, ptr addrspace(1) %valptr, i32 %tid
@@ -362,11 +362,11 @@ define amdgpu_kernel void @v_cttz_v4i32(ptr addrspace(1) noalias %out, ptr addrs
;
; GFX10-LABEL: v_cttz_v4i32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 4, v0
; GFX10-NEXT: v_mov_b32_e32 v4, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx4 v[0:3], v0, s[2:3]
+; GFX10-NEXT: global_load_dwordx4 v[0:3], v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbl_b32_e32 v3, v3
; GFX10-NEXT: v_ffbl_b32_e32 v2, v2
@@ -376,16 +376,16 @@ define amdgpu_kernel void @v_cttz_v4i32(ptr addrspace(1) noalias %out, ptr addrs
; GFX10-NEXT: v_min_u32_e32 v2, 32, v2
; GFX10-NEXT: v_min_u32_e32 v1, 32, v1
; GFX10-NEXT: v_min_u32_e32 v0, 32, v0
-; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_cttz_v4i32:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 4, v0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_dwordx4 v[0:3], v0, s[2:3]
+; GFX10-GISEL-NEXT: global_load_dwordx4 v[0:3], v0, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v0, v0
; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v1, v1
@@ -395,7 +395,7 @@ define amdgpu_kernel void @v_cttz_v4i32(ptr addrspace(1) noalias %out, ptr addrs
; GFX10-GISEL-NEXT: v_min_u32_e32 v1, 32, v1
; GFX10-GISEL-NEXT: v_min_u32_e32 v2, 32, v2
; GFX10-GISEL-NEXT: v_min_u32_e32 v3, 32, v3
-; GFX10-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX10-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr <4 x i32>, ptr addrspace(1) %valptr, i32 %tid
@@ -475,26 +475,26 @@ define amdgpu_kernel void @v_cttz_i8(ptr addrspace(1) noalias %out, ptr addrspac
;
; GFX10-LABEL: v_cttz_i8:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_ubyte v1, v0, s[2:3]
+; GFX10-NEXT: global_load_ubyte v1, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_or_b32_e32 v1, 0x100, v1
; GFX10-NEXT: v_ffbl_b32_e32 v1, v1
-; GFX10-NEXT: global_store_byte v0, v1, s[0:1]
+; GFX10-NEXT: global_store_byte v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_cttz_i8:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_ubyte v1, v0, s[2:3]
+; GFX10-GISEL-NEXT: global_load_ubyte v1, v0, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_or_b32_e32 v1, 0x100, v1
; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v1, v1
-; GFX10-GISEL-NEXT: global_store_byte v0, v1, s[0:1]
+; GFX10-GISEL-NEXT: global_store_byte v0, v1, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
%val = load i8, ptr addrspace(1) %valptr
%cttz = call i8 @llvm.cttz.i8(i8 %val, i1 false) nounwind readnone
@@ -629,24 +629,24 @@ define amdgpu_kernel void @s_cttz_i64_trunc(ptr addrspace(1) noalias %out, i64 %
;
; GFX10-LABEL: s_cttz_i64_trunc:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_ff1_i32_b64 s2, s[2:3]
-; GFX10-NEXT: s_min_u32 s2, s2, 64
-; GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_ff1_i32_b64 s0, s[6:7]
+; GFX10-NEXT: s_min_u32 s0, s0, 64
+; GFX10-NEXT: v_mov_b32_e32 v1, s0
+; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: s_cttz_i64_trunc:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: s_ff1_i32_b64 s2, s[2:3]
-; GFX10-GISEL-NEXT: s_min_u32 s2, s2, 64
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: s_ff1_i32_b64 s0, s[6:7]
+; GFX10-GISEL-NEXT: s_min_u32 s0, s0, 64
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
%cttz = call i64 @llvm.cttz.i64(i64 %val, i1 false)
%trunc = trunc i64 %cttz to i32
@@ -726,25 +726,25 @@ define amdgpu_kernel void @v_cttz_i64(ptr addrspace(1) noalias %out, ptr addrspa
;
; GFX10-LABEL: v_cttz_i64:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3]
+; GFX10-NEXT: global_load_dwordx2 v[0:1], v2, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbl_b32_e32 v1, v1
; GFX10-NEXT: v_ffbl_b32_e32 v0, v0
; GFX10-NEXT: v_add_nc_u32_e64 v1, v1, 32 clamp
; GFX10-NEXT: v_min3_u32 v0, v0, v1, 64
; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_cttz_i64:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3]
+; GFX10-GISEL-NEXT: global_load_dwordx2 v[0:1], v2, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v1, v1
; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v0, v0
@@ -752,7 +752,7 @@ define amdgpu_kernel void @v_cttz_i64(ptr addrspace(1) noalias %out, ptr addrspa
; GFX10-GISEL-NEXT: v_min_u32_e32 v0, v0, v1
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: v_min_u32_e32 v0, 64, v0
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr i64, ptr addrspace(1) %in, i32 %tid
@@ -835,33 +835,33 @@ define amdgpu_kernel void @v_cttz_i64_trunc(ptr addrspace(1) noalias %out, ptr a
;
; GFX10-LABEL: v_cttz_i64_trunc:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx2 v[1:2], v1, s[2:3]
+; GFX10-NEXT: global_load_dwordx2 v[1:2], v1, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbl_b32_e32 v2, v2
; GFX10-NEXT: v_ffbl_b32_e32 v1, v1
; GFX10-NEXT: v_add_nc_u32_e64 v2, v2, 32 clamp
; GFX10-NEXT: v_min3_u32 v1, v1, v2, 64
-; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_cttz_i64_trunc:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_dwordx2 v[1:2], v1, s[2:3]
+; GFX10-GISEL-NEXT: global_load_dwordx2 v[1:2], v1, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v2, v2
; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v1, v1
; GFX10-GISEL-NEXT: v_add_nc_u32_e64 v2, v2, 32 clamp
; GFX10-GISEL-NEXT: v_min_u32_e32 v1, v1, v2
; GFX10-GISEL-NEXT: v_min_u32_e32 v1, 64, v1
-; GFX10-GISEL-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-GISEL-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr i64, ptr addrspace(1) %in, i32 %tid
@@ -933,29 +933,29 @@ define amdgpu_kernel void @v_cttz_i32_sel_eq_neg1(ptr addrspace(1) noalias %out,
;
; GFX10-LABEL: v_cttz_i32_sel_eq_neg1:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbl_b32_e32 v0, v0
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_cttz_i32_sel_eq_neg1:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-GISEL-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v1, v0
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-GISEL-NEXT: v_min_u32_e32 v1, 32, v1
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, v1, -1, vcc_lo
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr i32, ptr addrspace(1) %valptr, i32 %tid
@@ -1027,29 +1027,29 @@ define amdgpu_kernel void @v_cttz_i32_sel_ne_neg1(ptr addrspace(1) noalias %out,
;
; GFX10-LABEL: v_cttz_i32_sel_ne_neg1:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbl_b32_e32 v0, v0
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_cttz_i32_sel_ne_neg1:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-GISEL-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v1, v0
; GFX10-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GFX10-GISEL-NEXT: v_min_u32_e32 v1, 32, v1
; GFX10-GISEL-NEXT: v_cndmask_b32_e32 v0, -1, v1, vcc_lo
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr i32, ptr addrspace(1) %valptr, i32 %tid
@@ -1130,32 +1130,32 @@ define amdgpu_kernel void @v_cttz_i32_sel_eq_bitwidth(ptr addrspace(1) noalias %
;
; GFX10-LABEL: v_cttz_i32_sel_eq_bitwidth:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbl_b32_e32 v0, v0
; GFX10-NEXT: v_min_u32_e32 v0, 32, v0
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 32, v0
; GFX10-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc_lo
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_cttz_i32_sel_eq_bitwidth:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-GISEL-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v0, v0
; GFX10-GISEL-NEXT: v_min_u32_e32 v0, 32, v0
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 32, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, v0, -1, vcc_lo
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr i32, ptr addrspace(1) %valptr, i32 %tid
@@ -1235,32 +1235,32 @@ define amdgpu_kernel void @v_cttz_i32_sel_ne_bitwidth(ptr addrspace(1) noalias %
;
; GFX10-LABEL: v_cttz_i32_sel_ne_bitwidth:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbl_b32_e32 v0, v0
; GFX10-NEXT: v_min_u32_e32 v0, 32, v0
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 32, v0
; GFX10-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc_lo
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_cttz_i32_sel_ne_bitwidth:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-GISEL-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v0, v0
; GFX10-GISEL-NEXT: v_min_u32_e32 v0, 32, v0
; GFX10-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, 32, v0
; GFX10-GISEL-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc_lo
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr i32, ptr addrspace(1) %valptr, i32 %tid
@@ -1335,32 +1335,32 @@ define amdgpu_kernel void @v_cttz_i32_sel_ne_bitwidth(ptr addrspace(1) noalias %
;
; GFX10-LABEL: v_cttz_i8_sel_eq_neg1:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_ubyte v0, v0, s[2:3]
+; GFX10-NEXT: global_load_ubyte v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbl_b32_e32 v0, v0
-; GFX10-NEXT: global_store_byte v1, v0, s[0:1]
+; GFX10-NEXT: global_store_byte v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_cttz_i8_sel_eq_neg1:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s7
; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v2, v3, vcc_lo
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-GISEL-NEXT: global_load_ubyte v0, v[0:1], off
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_or_b32_e32 v1, 0x100, v0
-; GFX10-GISEL-NEXT: v_cmp_eq_u32_sdwa s2, v0, v2 src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-GISEL-NEXT: v_cmp_eq_u32_sdwa s0, v0, v2 src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v1, v1
-; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, v1, -1, s2
-; GFX10-GISEL-NEXT: global_store_byte v2, v0, s[0:1]
+; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, v1, -1, s0
+; GFX10-GISEL-NEXT: global_store_byte v2, v0, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%valptr.gep = getelementptr i8, ptr addrspace(1) %valptr, i32 %tid
@@ -1442,31 +1442,31 @@ define amdgpu_kernel void @v_cttz_i32_sel_ne_bitwidth(ptr addrspace(1) noalias %
;
; GFX10-LABEL: v_cttz_i16_sel_eq_neg1:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_ushort v1, v0, s[2:3]
+; GFX10-NEXT: global_load_ushort v1, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_or_b32_e32 v2, 0x10000, v1
; GFX10-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v1
; GFX10-NEXT: v_ffbl_b32_e32 v2, v2
; GFX10-NEXT: v_cndmask_b32_e32 v1, 0xffff, v2, vcc_lo
-; GFX10-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_cttz_i16_sel_eq_neg1:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: global_load_ushort v1, v0, s[2:3]
+; GFX10-GISEL-NEXT: global_load_ushort v1, v0, s[6:7]
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_or_b32_e32 v2, 0x10000, v1
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX10-GISEL-NEXT: v_ffbl_b32_e32 v2, v2
; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v1, v2, 0xffff, vcc_lo
-; GFX10-GISEL-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-GISEL-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
%val = load i16, ptr addrspace(1) %valptr
%cttz = call i16 @llvm.cttz.i16(i16 %val, i1 false) nounwind readnone
@@ -1542,23 +1542,23 @@ define amdgpu_kernel void @v_cttz_i7_sel_eq_neg1(ptr addrspace(1) noalias %out,
;
; GFX10-LABEL: v_cttz_i7_sel_eq_neg1:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_ubyte v0, v0, s[2:3]
+; GFX10-NEXT: global_load_ubyte v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_ffbl_b32_e32 v0, v0
; GFX10-NEXT: v_and_b32_e32 v0, 0x7f, v0
-; GFX10-NEXT: global_store_byte v1, v0, s[0:1]
+; GFX10-NEXT: global_store_byte v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_cttz_i7_sel_eq_neg1:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s7
; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v1, v0
; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v2, v3, vcc_lo
; GFX10-GISEL-NEXT: global_load_ubyte v0, v[0:1], off
@@ -1570,7 +1570,7 @@ define amdgpu_kernel void @v_cttz_i7_sel_eq_neg1(ptr addrspace(1) noalias %out,
; GFX10-GISEL-NEXT: v_cndmask_b32_e64 v0, v1, 0x7f, vcc_lo
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0x7f, v0
-; GFX10-GISEL-NEXT: global_store_byte v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: global_store_byte v1, v0, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%valptr.gep = getelementptr i7, ptr addrspace(1) %valptr, i32 %tid
diff --git a/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll b/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
index 2491abe4bc1ce..81ed823bad204 100644
--- a/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll
@@ -121,14 +121,14 @@ define amdgpu_kernel void @v_cttz_zero_undef_i32(ptr addrspace(1) noalias %out,
;
; GFX9-GISEL-LABEL: v_cttz_zero_undef_i32:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX9-GISEL-NEXT: global_load_dword v0, v0, s[6:7]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_ffbl_b32_e32 v0, v0
-; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr i32, ptr addrspace(1) %valptr, i32 %tid
@@ -198,15 +198,15 @@ define amdgpu_kernel void @v_cttz_zero_undef_v2i32(ptr addrspace(1) noalias %out
;
; GFX9-GISEL-LABEL: v_cttz_zero_undef_v2i32:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dwordx2 v[0:1], v0, s[2:3]
+; GFX9-GISEL-NEXT: global_load_dwordx2 v[0:1], v0, s[6:7]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_ffbl_b32_e32 v0, v0
; GFX9-GISEL-NEXT: v_ffbl_b32_e32 v1, v1
-; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr <2 x i32>, ptr addrspace(1) %valptr, i32 %tid
@@ -282,17 +282,17 @@ define amdgpu_kernel void @v_cttz_zero_undef_v4i32(ptr addrspace(1) noalias %out
;
; GFX9-GISEL-LABEL: v_cttz_zero_undef_v4i32:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v0, 4, v0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_dwordx4 v[0:3], v0, s[2:3]
+; GFX9-GISEL-NEXT: global_load_dwordx4 v[0:3], v0, s[6:7]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_ffbl_b32_e32 v0, v0
; GFX9-GISEL-NEXT: v_ffbl_b32_e32 v1, v1
; GFX9-GISEL-NEXT: v_ffbl_b32_e32 v2, v2
; GFX9-GISEL-NEXT: v_ffbl_b32_e32 v3, v3
-; GFX9-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX9-GISEL-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%in.gep = getelementptr <4 x i32>, ptr addrspace(1) %valptr, i32 %tid
@@ -538,14 +538,14 @@ define amdgpu_kernel void @s_cttz_zero_undef_i64_with_select(ptr addrspace(1) no
;
; GFX9-GISEL-LABEL: s_cttz_zero_undef_i64_with_select:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX9-GISEL-NEXT: s_mov_b32 s5, 0
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_mov_b32 s1, 0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_ff1_i32_b64 s4, s[2:3]
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_ff1_i32_b64 s0, s[6:7]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%cttz = tail call i64 @llvm.cttz.i64(i64 %val, i1 true) nounwind readnone
%cttz_ret = icmp ne i64 %val, 0
@@ -622,16 +622,16 @@ define amdgpu_kernel void @v_cttz_zero_undef_i8_with_select(ptr addrspace(1) noa
;
; GFX9-GISEL-LABEL: v_cttz_zero_undef_i8_with_select:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[2:3]
+; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[6:7]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_ffbl_b32_e32 v2, v1
; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
; GFX9-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, 32, v2, vcc
-; GFX9-GISEL-NEXT: global_store_byte v0, v1, s[0:1]
+; GFX9-GISEL-NEXT: global_store_byte v0, v1, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%val = load i8, ptr addrspace(1) %arrayidx, align 1
%cttz = tail call i8 @llvm.cttz.i8(i8 %val, i1 true) nounwind readnone
@@ -721,18 +721,18 @@ define amdgpu_kernel void @v_cttz_zero_undef_i16_with_select(ptr addrspace(1) no
;
; GFX9-GISEL-LABEL: v_cttz_zero_undef_i16_with_select:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-GISEL-NEXT: global_load_ubyte v2, v0, s[2:3] offset:1
+; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[6:7]
+; GFX9-GISEL-NEXT: global_load_ubyte v2, v0, s[6:7] offset:1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v2, 8, v1
; GFX9-GISEL-NEXT: v_ffbl_b32_e32 v2, v1
; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX9-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, 32, v2, vcc
-; GFX9-GISEL-NEXT: global_store_short v0, v1, s[0:1]
+; GFX9-GISEL-NEXT: global_store_short v0, v1, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%val = load i16, ptr addrspace(1) %arrayidx, align 1
%cttz = tail call i16 @llvm.cttz.i16(i16 %val, i1 true) nounwind readnone
@@ -836,13 +836,13 @@ define amdgpu_kernel void @v_cttz_zero_undef_i32_with_select(ptr addrspace(1) no
;
; GFX9-GISEL-LABEL: v_cttz_zero_undef_i32_with_select:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-GISEL-NEXT: global_load_ubyte v2, v0, s[2:3] offset:1
-; GFX9-GISEL-NEXT: global_load_ubyte v3, v0, s[2:3] offset:3
-; GFX9-GISEL-NEXT: global_load_ubyte v4, v0, s[2:3] offset:2
+; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[6:7]
+; GFX9-GISEL-NEXT: global_load_ubyte v2, v0, s[6:7] offset:1
+; GFX9-GISEL-NEXT: global_load_ubyte v3, v0, s[6:7] offset:3
+; GFX9-GISEL-NEXT: global_load_ubyte v4, v0, s[6:7] offset:2
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(2)
; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v2, 8, v1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(1)
@@ -853,7 +853,7 @@ define amdgpu_kernel void @v_cttz_zero_undef_i32_with_select(ptr addrspace(1) no
; GFX9-GISEL-NEXT: v_ffbl_b32_e32 v2, v1
; GFX9-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, 32, v2, vcc
-; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%val = load i32, ptr addrspace(1) %arrayidx, align 1
%cttz = tail call i32 @llvm.cttz.i32(i32 %val, i1 true) nounwind readnone
@@ -1017,17 +1017,17 @@ define amdgpu_kernel void @v_cttz_zero_undef_i64_with_select(ptr addrspace(1) no
;
; GFX9-GISEL-LABEL: v_cttz_zero_undef_i64_with_select:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_ubyte v0, v1, s[2:3]
-; GFX9-GISEL-NEXT: global_load_ubyte v2, v1, s[2:3] offset:1
-; GFX9-GISEL-NEXT: global_load_ubyte v3, v1, s[2:3] offset:2
-; GFX9-GISEL-NEXT: global_load_ubyte v4, v1, s[2:3] offset:3
-; GFX9-GISEL-NEXT: global_load_ubyte v5, v1, s[2:3] offset:4
-; GFX9-GISEL-NEXT: global_load_ubyte v6, v1, s[2:3] offset:5
-; GFX9-GISEL-NEXT: global_load_ubyte v7, v1, s[2:3] offset:6
-; GFX9-GISEL-NEXT: global_load_ubyte v8, v1, s[2:3] offset:7
+; GFX9-GISEL-NEXT: global_load_ubyte v0, v1, s[6:7]
+; GFX9-GISEL-NEXT: global_load_ubyte v2, v1, s[6:7] offset:1
+; GFX9-GISEL-NEXT: global_load_ubyte v3, v1, s[6:7] offset:2
+; GFX9-GISEL-NEXT: global_load_ubyte v4, v1, s[6:7] offset:3
+; GFX9-GISEL-NEXT: global_load_ubyte v5, v1, s[6:7] offset:4
+; GFX9-GISEL-NEXT: global_load_ubyte v6, v1, s[6:7] offset:5
+; GFX9-GISEL-NEXT: global_load_ubyte v7, v1, s[6:7] offset:6
+; GFX9-GISEL-NEXT: global_load_ubyte v8, v1, s[6:7] offset:7
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(6)
; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v2, 8, v0
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(5)
@@ -1048,7 +1048,7 @@ define amdgpu_kernel void @v_cttz_zero_undef_i64_with_select(ptr addrspace(1) no
; GFX9-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX9-GISEL-NEXT: v_min_u32_e32 v0, v0, v4
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, 64, v0, vcc
-; GFX9-GISEL-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: global_store_dwordx2 v1, v[0:1], s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%val = load i64, ptr addrspace(1) %arrayidx, align 1
%cttz = tail call i64 @llvm.cttz.i64(i64 %val, i1 true) nounwind readnone
@@ -1152,13 +1152,13 @@ define amdgpu_kernel void @v_cttz_i32_sel_eq_neg1(ptr addrspace(1) noalias %out,
;
; GFX9-GISEL-LABEL: v_cttz_i32_sel_eq_neg1:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-GISEL-NEXT: global_load_ubyte v2, v0, s[2:3] offset:1
-; GFX9-GISEL-NEXT: global_load_ubyte v3, v0, s[2:3] offset:3
-; GFX9-GISEL-NEXT: global_load_ubyte v4, v0, s[2:3] offset:2
+; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[6:7]
+; GFX9-GISEL-NEXT: global_load_ubyte v2, v0, s[6:7] offset:1
+; GFX9-GISEL-NEXT: global_load_ubyte v3, v0, s[6:7] offset:3
+; GFX9-GISEL-NEXT: global_load_ubyte v4, v0, s[6:7] offset:2
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(2)
; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v2, 8, v1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(1)
@@ -1170,7 +1170,7 @@ define amdgpu_kernel void @v_cttz_i32_sel_eq_neg1(ptr addrspace(1) noalias %out,
; GFX9-GISEL-NEXT: v_min_u32_e32 v2, 32, v2
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v1, v2, -1, vcc
-; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%val = load i32, ptr addrspace(1) %arrayidx, align 1
%ctlz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone
@@ -1274,13 +1274,13 @@ define amdgpu_kernel void @v_cttz_i32_sel_ne_neg1(ptr addrspace(1) noalias %out,
;
; GFX9-GISEL-LABEL: v_cttz_i32_sel_ne_neg1:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-GISEL-NEXT: global_load_ubyte v2, v0, s[2:3] offset:1
-; GFX9-GISEL-NEXT: global_load_ubyte v3, v0, s[2:3] offset:3
-; GFX9-GISEL-NEXT: global_load_ubyte v4, v0, s[2:3] offset:2
+; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[6:7]
+; GFX9-GISEL-NEXT: global_load_ubyte v2, v0, s[6:7] offset:1
+; GFX9-GISEL-NEXT: global_load_ubyte v3, v0, s[6:7] offset:3
+; GFX9-GISEL-NEXT: global_load_ubyte v4, v0, s[6:7] offset:2
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(2)
; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v2, 8, v1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(1)
@@ -1292,7 +1292,7 @@ define amdgpu_kernel void @v_cttz_i32_sel_ne_neg1(ptr addrspace(1) noalias %out,
; GFX9-GISEL-NEXT: v_min_u32_e32 v2, 32, v2
; GFX9-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, -1, v2, vcc
-; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%val = load i32, ptr addrspace(1) %arrayidx, align 1
%ctlz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone
@@ -1404,13 +1404,13 @@ define amdgpu_kernel void @v_cttz_i32_sel_ne_bitwidth(ptr addrspace(1) noalias %
;
; GFX9-GISEL-LABEL: v_cttz_i32_sel_ne_bitwidth:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-GISEL-NEXT: global_load_ubyte v2, v0, s[2:3] offset:1
-; GFX9-GISEL-NEXT: global_load_ubyte v3, v0, s[2:3] offset:3
-; GFX9-GISEL-NEXT: global_load_ubyte v4, v0, s[2:3] offset:2
+; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[6:7]
+; GFX9-GISEL-NEXT: global_load_ubyte v2, v0, s[6:7] offset:1
+; GFX9-GISEL-NEXT: global_load_ubyte v3, v0, s[6:7] offset:3
+; GFX9-GISEL-NEXT: global_load_ubyte v4, v0, s[6:7] offset:2
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(2)
; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v2, 8, v1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(1)
@@ -1422,7 +1422,7 @@ define amdgpu_kernel void @v_cttz_i32_sel_ne_bitwidth(ptr addrspace(1) noalias %
; GFX9-GISEL-NEXT: v_min_u32_e32 v1, 32, v1
; GFX9-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 32, v1
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, -1, v1, vcc
-; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-GISEL-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%val = load i32, ptr addrspace(1) %arrayidx, align 1
%ctlz = call i32 @llvm.cttz.i32(i32 %val, i1 false) nounwind readnone
@@ -1498,18 +1498,18 @@ define amdgpu_kernel void @v_cttz_i32_sel_ne_bitwidth(ptr addrspace(1) noalias %
;
; GFX9-GISEL-LABEL: v_cttz_i8_sel_eq_neg1:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xff
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[2:3]
+; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[6:7]
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_or_b32_e32 v3, 0x100, v1
; GFX9-GISEL-NEXT: v_ffbl_b32_e32 v3, v3
; GFX9-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
-; GFX9-GISEL-NEXT: global_store_byte v0, v1, s[0:1]
+; GFX9-GISEL-NEXT: global_store_byte v0, v1, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%val = load i8, ptr addrspace(1) %arrayidx, align 1
%ctlz = call i8 @llvm.cttz.i8(i8 %val, i1 false) nounwind readnone
@@ -1597,12 +1597,12 @@ define amdgpu_kernel void @v_cttz_i32_sel_ne_bitwidth(ptr addrspace(1) noalias %
;
; GFX9-GISEL-LABEL: v_cttz_i16_sel_eq_neg1:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0xffff
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[2:3]
-; GFX9-GISEL-NEXT: global_load_ubyte v2, v0, s[2:3] offset:1
+; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[6:7]
+; GFX9-GISEL-NEXT: global_load_ubyte v2, v0, s[6:7] offset:1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v2, 8, v1
; GFX9-GISEL-NEXT: v_or_b32_e32 v2, 0x10000, v1
@@ -1610,7 +1610,7 @@ define amdgpu_kernel void @v_cttz_i32_sel_ne_bitwidth(ptr addrspace(1) noalias %
; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
-; GFX9-GISEL-NEXT: global_store_short v0, v1, s[0:1]
+; GFX9-GISEL-NEXT: global_store_short v0, v1, s[4:5]
; GFX9-GISEL-NEXT: s_endpgm
%val = load i16, ptr addrspace(1) %arrayidx, align 1
%ctlz = call i16 @llvm.cttz.i16(i16 %val, i1 false) nounwind readnone
diff --git a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
index 96969a12b2c58..eb959e30b87f4 100644
--- a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
@@ -933,24 +933,24 @@ define amdgpu_kernel void @load_i8_to_f32(ptr addrspace(1) noalias %out, ptr add
;
; GFX10-LABEL: load_i8_to_f32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_ubyte v0, v0, s[2:3]
+; GFX10-NEXT: global_load_ubyte v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX9-LABEL: load_i8_to_f32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ubyte v0, v0, s[2:3]
+; GFX9-NEXT: global_load_ubyte v0, v0, s[6:7]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: load_i8_to_f32:
@@ -1013,28 +1013,28 @@ define amdgpu_kernel void @load_v2i8_to_v2f32(ptr addrspace(1) noalias %out, ptr
;
; GFX10-LABEL: load_v2i8_to_v2f32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_ushort v0, v0, s[2:3]
+; GFX10-NEXT: global_load_ushort v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v1, v0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX9-LABEL: load_v2i8_to_v2f32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ushort v0, v0, s[2:3]
+; GFX9-NEXT: global_load_ushort v0, v0, s[6:7]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte1_e32 v1, v0
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: load_v2i8_to_v2f32:
@@ -1104,30 +1104,30 @@ define amdgpu_kernel void @load_v3i8_to_v3f32(ptr addrspace(1) noalias %out, ptr
;
; GFX10-LABEL: load_v3i8_to_v3f32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v2, v0
; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v1, v0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; GFX10-NEXT: global_store_dwordx3 v3, v[0:2], s[0:1]
+; GFX10-NEXT: global_store_dwordx3 v3, v[0:2], s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX9-LABEL: load_v3i8_to_v3f32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX9-NEXT: global_load_dword v0, v0, s[6:7]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte2_e32 v2, v0
; GFX9-NEXT: v_cvt_f32_ubyte1_e32 v1, v0
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; GFX9-NEXT: global_store_dwordx3 v3, v[0:2], s[0:1]
+; GFX9-NEXT: global_store_dwordx3 v3, v[0:2], s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: load_v3i8_to_v3f32:
@@ -1198,32 +1198,32 @@ define amdgpu_kernel void @load_v4i8_to_v4f32(ptr addrspace(1) noalias %out, ptr
;
; GFX10-LABEL: load_v4i8_to_v4f32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v4, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v3, v0
; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v2, v0
; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v1, v0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX9-LABEL: load_v4i8_to_v4f32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX9-NEXT: global_load_dword v0, v0, s[6:7]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte3_e32 v3, v0
; GFX9-NEXT: v_cvt_f32_ubyte2_e32 v2, v0
; GFX9-NEXT: v_cvt_f32_ubyte1_e32 v1, v0
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: load_v4i8_to_v4f32:
@@ -1318,15 +1318,15 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_unaligned(ptr addrspace(1) noalias
;
; GFX10-LABEL: load_v4i8_to_v4f32_unaligned:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v6, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_clause 0x3
-; GFX10-NEXT: global_load_ubyte v1, v0, s[2:3] offset:3
-; GFX10-NEXT: global_load_ubyte v2, v0, s[2:3] offset:2
-; GFX10-NEXT: global_load_ubyte v4, v0, s[2:3] offset:1
-; GFX10-NEXT: global_load_ubyte v5, v0, s[2:3]
+; GFX10-NEXT: global_load_ubyte v1, v0, s[6:7] offset:3
+; GFX10-NEXT: global_load_ubyte v2, v0, s[6:7] offset:2
+; GFX10-NEXT: global_load_ubyte v4, v0, s[6:7] offset:1
+; GFX10-NEXT: global_load_ubyte v5, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(3)
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v3, v1
; GFX10-NEXT: s_waitcnt vmcnt(2)
@@ -1335,19 +1335,19 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_unaligned(ptr addrspace(1) noalias
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v5
-; GFX10-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
+; GFX10-NEXT: global_store_dwordx4 v6, v[0:3], s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX9-LABEL: load_v4i8_to_v4f32_unaligned:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ubyte v1, v0, s[2:3] offset:3
-; GFX9-NEXT: global_load_ubyte v2, v0, s[2:3] offset:2
-; GFX9-NEXT: global_load_ubyte v4, v0, s[2:3] offset:1
-; GFX9-NEXT: global_load_ubyte v5, v0, s[2:3]
+; GFX9-NEXT: global_load_ubyte v1, v0, s[6:7] offset:3
+; GFX9-NEXT: global_load_ubyte v2, v0, s[6:7] offset:2
+; GFX9-NEXT: global_load_ubyte v4, v0, s[6:7] offset:1
+; GFX9-NEXT: global_load_ubyte v5, v0, s[6:7]
; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v3, v1
; GFX9-NEXT: s_waitcnt vmcnt(2)
@@ -1356,7 +1356,7 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_unaligned(ptr addrspace(1) noalias
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v1, v4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, v5
-; GFX9-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
+; GFX9-NEXT: global_store_dwordx4 v6, v[0:3], s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: load_v4i8_to_v4f32_unaligned:
@@ -1481,15 +1481,15 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_unaligned_multiuse(ptr addrspace(1
;
; GFX10-LABEL: load_v4i8_to_v4f32_unaligned_multiuse:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v7, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_clause 0x3
-; GFX10-NEXT: global_load_ubyte v1, v0, s[4:5] offset:2
-; GFX10-NEXT: global_load_ubyte v3, v0, s[4:5] offset:3
-; GFX10-NEXT: global_load_ubyte v2, v0, s[6:7] offset:3
-; GFX10-NEXT: global_load_ubyte v4, v0, s[6:7] offset:2
+; GFX10-NEXT: global_load_ubyte v1, v0, s[8:9] offset:2
+; GFX10-NEXT: global_load_ubyte v3, v0, s[8:9] offset:3
+; GFX10-NEXT: global_load_ubyte v2, v0, s[10:11] offset:3
+; GFX10-NEXT: global_load_ubyte v4, v0, s[10:11] offset:2
; GFX10-NEXT: s_waitcnt vmcnt(2)
; GFX10-NEXT: v_lshl_or_b32 v5, v3, 8, v1
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
@@ -1499,21 +1499,21 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_unaligned_multiuse(ptr addrspace(1
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v3
; GFX10-NEXT: v_mov_b32_e32 v3, v1
; GFX10-NEXT: v_perm_b32 v4, v5, v6, 0x4000405
-; GFX10-NEXT: global_store_dwordx4 v7, v[0:3], s[0:1]
-; GFX10-NEXT: global_store_dword v7, v4, s[2:3]
+; GFX10-NEXT: global_store_dwordx4 v7, v[0:3], s[4:5]
+; GFX10-NEXT: global_store_dword v7, v4, s[6:7]
; GFX10-NEXT: s_endpgm
;
; GFX9-LABEL: load_v4i8_to_v4f32_unaligned_multiuse:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v5, 0
+; GFX9-NEXT: s_mov_b32 s0, 0x4000405
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ubyte v1, v0, s[4:5] offset:2
-; GFX9-NEXT: global_load_ubyte v2, v0, s[6:7] offset:3
-; GFX9-NEXT: global_load_ubyte v3, v0, s[4:5] offset:3
-; GFX9-NEXT: global_load_ubyte v4, v0, s[6:7] offset:2
-; GFX9-NEXT: s_mov_b32 s4, 0x4000405
+; GFX9-NEXT: global_load_ubyte v1, v0, s[8:9] offset:2
+; GFX9-NEXT: global_load_ubyte v2, v0, s[10:11] offset:3
+; GFX9-NEXT: global_load_ubyte v3, v0, s[8:9] offset:3
+; GFX9-NEXT: global_load_ubyte v4, v0, s[10:11] offset:2
; GFX9-NEXT: s_waitcnt vmcnt(1)
; GFX9-NEXT: v_lshl_or_b32 v6, v3, 8, v1
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
@@ -1522,9 +1522,9 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_unaligned_multiuse(ptr addrspace(1
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v2, v4
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, v3
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_perm_b32 v4, v6, v7, s4
-; GFX9-NEXT: global_store_dwordx4 v5, v[0:3], s[0:1]
-; GFX9-NEXT: global_store_dword v5, v4, s[2:3]
+; GFX9-NEXT: v_perm_b32 v4, v6, v7, s0
+; GFX9-NEXT: global_store_dwordx4 v5, v[0:3], s[4:5]
+; GFX9-NEXT: global_store_dword v5, v4, s[6:7]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: load_v4i8_to_v4f32_unaligned_multiuse:
@@ -1653,12 +1653,12 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_2_uses(ptr addrspace(1) noalias %o
;
; GFX10-LABEL: load_v4i8_to_v4f32_2_uses:
; GFX10: ; %bb.0:
+; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x34
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: global_load_dword v0, v0, s[0:1]
-; GFX10-NEXT: s_waitcnt_depctr 0xffe3
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX10-NEXT: v_and_b32_e32 v3, 0xffffff00, v0
@@ -1676,22 +1676,21 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_2_uses(ptr addrspace(1) noalias %o
; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v1, v0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX10-NEXT: v_or_b32_sdwa v5, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
-; GFX10-NEXT: global_store_dword v4, v5, s[2:3]
+; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
+; GFX10-NEXT: global_store_dword v4, v5, s[6:7]
; GFX10-NEXT: s_endpgm
;
; GFX9-LABEL: load_v4i8_to_v4f32_2_uses:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX9-NEXT: s_movk_i32 s4, 0xff00
; GFX9-NEXT: v_mov_b32_e32 v6, 9
; GFX9-NEXT: v_mov_b32_e32 v5, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v4, v0, s[0:1]
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX9-NEXT: s_movk_i32 s5, 0x900
+; GFX9-NEXT: s_movk_i32 s0, 0xff00
+; GFX9-NEXT: s_movk_i32 s1, 0x900
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte3_e32 v3, v4
; GFX9-NEXT: v_cvt_f32_ubyte2_e32 v2, v4
@@ -1699,17 +1698,16 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_2_uses(ptr addrspace(1) noalias %o
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, v4
; GFX9-NEXT: v_and_b32_e32 v7, 0xffffff00, v4
; GFX9-NEXT: v_add_u16_e32 v8, 9, v4
-; GFX9-NEXT: v_and_b32_sdwa v9, v4, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT: v_and_b32_sdwa v9, v4, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX9-NEXT: v_add_u16_sdwa v4, v4, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_dwordx4 v5, v[0:3], s[0:1]
+; GFX9-NEXT: global_store_dwordx4 v5, v[0:3], s[4:5]
; GFX9-NEXT: s_nop 0
; GFX9-NEXT: v_or_b32_sdwa v0, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: v_or_b32_sdwa v1, v9, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: v_add_u16_e32 v0, 0x900, v0
-; GFX9-NEXT: v_add_u16_sdwa v1, v1, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT: v_add_u16_sdwa v1, v1, s1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-NEXT: global_store_dword v5, v0, s[2:3]
+; GFX9-NEXT: global_store_dword v5, v0, s[6:7]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: load_v4i8_to_v4f32_2_uses:
@@ -1851,17 +1849,17 @@ define amdgpu_kernel void @load_v7i8_to_v7f32(ptr addrspace(1) noalias %out, ptr
;
; GFX10-LABEL: load_v7i8_to_v7f32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX10-NEXT: v_mov_b32_e32 v8, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_clause 0x5
-; GFX10-NEXT: global_load_ubyte v4, v0, s[2:3] offset:6
-; GFX10-NEXT: global_load_ubyte v1, v0, s[2:3] offset:3
-; GFX10-NEXT: global_load_ubyte v2, v0, s[2:3] offset:2
-; GFX10-NEXT: global_load_ubyte v5, v0, s[2:3] offset:1
-; GFX10-NEXT: global_load_short_d16 v7, v0, s[2:3] offset:4
-; GFX10-NEXT: global_load_ubyte v0, v0, s[2:3]
+; GFX10-NEXT: global_load_ubyte v4, v0, s[6:7] offset:6
+; GFX10-NEXT: global_load_ubyte v1, v0, s[6:7] offset:3
+; GFX10-NEXT: global_load_ubyte v2, v0, s[6:7] offset:2
+; GFX10-NEXT: global_load_ubyte v5, v0, s[6:7] offset:1
+; GFX10-NEXT: global_load_short_d16 v7, v0, s[6:7] offset:4
+; GFX10-NEXT: global_load_ubyte v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(5)
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v6, v4
; GFX10-NEXT: s_waitcnt vmcnt(4)
@@ -1875,22 +1873,22 @@ define amdgpu_kernel void @load_v7i8_to_v7f32(ptr addrspace(1) noalias %out, ptr
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v4, v7
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; GFX10-NEXT: global_store_dwordx3 v8, v[4:6], s[0:1] offset:16
-; GFX10-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX10-NEXT: global_store_dwordx3 v8, v[4:6], s[4:5] offset:16
+; GFX10-NEXT: global_store_dwordx4 v8, v[0:3], s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX9-LABEL: load_v7i8_to_v7f32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX9-NEXT: v_mov_b32_e32 v10, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ubyte v1, v0, s[2:3] offset:6
-; GFX9-NEXT: global_load_ushort v2, v0, s[2:3] offset:4
-; GFX9-NEXT: global_load_ubyte v3, v0, s[2:3] offset:3
-; GFX9-NEXT: global_load_ubyte v7, v0, s[2:3] offset:2
-; GFX9-NEXT: global_load_ubyte v8, v0, s[2:3] offset:1
-; GFX9-NEXT: global_load_ubyte v9, v0, s[2:3]
+; GFX9-NEXT: global_load_ubyte v1, v0, s[6:7] offset:6
+; GFX9-NEXT: global_load_ushort v2, v0, s[6:7] offset:4
+; GFX9-NEXT: global_load_ubyte v3, v0, s[6:7] offset:3
+; GFX9-NEXT: global_load_ubyte v7, v0, s[6:7] offset:2
+; GFX9-NEXT: global_load_ubyte v8, v0, s[6:7] offset:1
+; GFX9-NEXT: global_load_ubyte v9, v0, s[6:7]
; GFX9-NEXT: s_waitcnt vmcnt(5)
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v6, v1
; GFX9-NEXT: s_waitcnt vmcnt(4)
@@ -1904,8 +1902,8 @@ define amdgpu_kernel void @load_v7i8_to_v7f32(ptr addrspace(1) noalias %out, ptr
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v1, v8
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, v9
-; GFX9-NEXT: global_store_dwordx4 v10, v[0:3], s[0:1]
-; GFX9-NEXT: global_store_dwordx3 v10, v[4:6], s[0:1] offset:16
+; GFX9-NEXT: global_store_dwordx4 v10, v[0:3], s[4:5]
+; GFX9-NEXT: global_store_dwordx3 v10, v[4:6], s[4:5] offset:16
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: load_v7i8_to_v7f32:
@@ -2004,11 +2002,11 @@ define amdgpu_kernel void @load_v8i8_to_v8f32(ptr addrspace(1) noalias %out, ptr
;
; GFX10-LABEL: load_v8i8_to_v8f32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX10-NEXT: v_mov_b32_e32 v10, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dwordx2 v[8:9], v0, s[2:3]
+; GFX10-NEXT: global_load_dwordx2 v[8:9], v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v7, v9
; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v6, v9
@@ -2018,17 +2016,17 @@ define amdgpu_kernel void @load_v8i8_to_v8f32(ptr addrspace(1) noalias %out, ptr
; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v2, v8
; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v1, v8
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v8
-; GFX10-NEXT: global_store_dwordx4 v10, v[4:7], s[0:1] offset:16
-; GFX10-NEXT: global_store_dwordx4 v10, v[0:3], s[0:1]
+; GFX10-NEXT: global_store_dwordx4 v10, v[4:7], s[4:5] offset:16
+; GFX10-NEXT: global_store_dwordx4 v10, v[0:3], s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX9-LABEL: load_v8i8_to_v8f32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX9-NEXT: v_mov_b32_e32 v9, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dwordx2 v[7:8], v0, s[2:3]
+; GFX9-NEXT: global_load_dwordx2 v[7:8], v0, s[6:7]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte3_e32 v3, v7
; GFX9-NEXT: v_cvt_f32_ubyte2_e32 v2, v7
@@ -2038,8 +2036,8 @@ define amdgpu_kernel void @load_v8i8_to_v8f32(ptr addrspace(1) noalias %out, ptr
; GFX9-NEXT: v_cvt_f32_ubyte2_e32 v6, v8
; GFX9-NEXT: v_cvt_f32_ubyte1_e32 v5, v8
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v4, v8
-; GFX9-NEXT: global_store_dwordx4 v9, v[4:7], s[0:1] offset:16
-; GFX9-NEXT: global_store_dwordx4 v9, v[0:3], s[0:1]
+; GFX9-NEXT: global_store_dwordx4 v9, v[4:7], s[4:5] offset:16
+; GFX9-NEXT: global_store_dwordx4 v9, v[0:3], s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: load_v8i8_to_v8f32:
@@ -2114,28 +2112,28 @@ define amdgpu_kernel void @i8_zext_inreg_i32_to_f32(ptr addrspace(1) noalias %ou
;
; GFX10-LABEL: i8_zext_inreg_i32_to_f32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_add_nc_u32_e32 v0, 2, v0
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX9-LABEL: i8_zext_inreg_i32_to_f32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX9-NEXT: global_load_dword v0, v0, s[6:7]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_add_u32_e32 v0, 2, v0
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: i8_zext_inreg_i32_to_f32:
@@ -2201,26 +2199,26 @@ define amdgpu_kernel void @i8_zext_inreg_hi1_to_f32(ptr addrspace(1) noalias %ou
;
; GFX10-LABEL: i8_zext_inreg_hi1_to_f32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v0, v0
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX9-LABEL: i8_zext_inreg_hi1_to_f32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX9-NEXT: global_load_dword v0, v0, s[6:7]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte1_e32 v0, v0
-; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: i8_zext_inreg_hi1_to_f32:
@@ -2285,24 +2283,24 @@ define amdgpu_kernel void @i8_zext_i32_to_f32(ptr addrspace(1) noalias %out, ptr
;
; GFX10-LABEL: i8_zext_i32_to_f32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_ubyte v0, v0, s[2:3]
+; GFX10-NEXT: global_load_ubyte v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX9-LABEL: i8_zext_i32_to_f32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ubyte v0, v0, s[2:3]
+; GFX9-NEXT: global_load_ubyte v0, v0, s[6:7]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: i8_zext_i32_to_f32:
@@ -2388,15 +2386,15 @@ define amdgpu_kernel void @v4i8_zext_v4i32_to_v4f32(ptr addrspace(1) noalias %ou
;
; GFX10-LABEL: v4i8_zext_v4i32_to_v4f32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v6, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_clause 0x3
-; GFX10-NEXT: global_load_ubyte v1, v0, s[2:3] offset:3
-; GFX10-NEXT: global_load_ubyte v2, v0, s[2:3] offset:2
-; GFX10-NEXT: global_load_ubyte v4, v0, s[2:3] offset:1
-; GFX10-NEXT: global_load_ubyte v5, v0, s[2:3]
+; GFX10-NEXT: global_load_ubyte v1, v0, s[6:7] offset:3
+; GFX10-NEXT: global_load_ubyte v2, v0, s[6:7] offset:2
+; GFX10-NEXT: global_load_ubyte v4, v0, s[6:7] offset:1
+; GFX10-NEXT: global_load_ubyte v5, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(3)
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v3, v1
; GFX10-NEXT: s_waitcnt vmcnt(2)
@@ -2405,19 +2403,19 @@ define amdgpu_kernel void @v4i8_zext_v4i32_to_v4f32(ptr addrspace(1) noalias %ou
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v5
-; GFX10-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
+; GFX10-NEXT: global_store_dwordx4 v6, v[0:3], s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX9-LABEL: v4i8_zext_v4i32_to_v4f32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ubyte v1, v0, s[2:3] offset:3
-; GFX9-NEXT: global_load_ubyte v2, v0, s[2:3] offset:2
-; GFX9-NEXT: global_load_ubyte v4, v0, s[2:3] offset:1
-; GFX9-NEXT: global_load_ubyte v5, v0, s[2:3]
+; GFX9-NEXT: global_load_ubyte v1, v0, s[6:7] offset:3
+; GFX9-NEXT: global_load_ubyte v2, v0, s[6:7] offset:2
+; GFX9-NEXT: global_load_ubyte v4, v0, s[6:7] offset:1
+; GFX9-NEXT: global_load_ubyte v5, v0, s[6:7]
; GFX9-NEXT: s_waitcnt vmcnt(3)
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v3, v1
; GFX9-NEXT: s_waitcnt vmcnt(2)
@@ -2426,7 +2424,7 @@ define amdgpu_kernel void @v4i8_zext_v4i32_to_v4f32(ptr addrspace(1) noalias %ou
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v1, v4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, v5
-; GFX9-NEXT: global_store_dwordx4 v6, v[0:3], s[0:1]
+; GFX9-NEXT: global_store_dwordx4 v6, v[0:3], s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: v4i8_zext_v4i32_to_v4f32:
@@ -2500,26 +2498,26 @@ define amdgpu_kernel void @extract_byte0_to_f32(ptr addrspace(1) noalias %out, p
;
; GFX10-LABEL: extract_byte0_to_f32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX9-LABEL: extract_byte0_to_f32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX9-NEXT: global_load_dword v0, v0, s[6:7]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: extract_byte0_to_f32:
@@ -2583,26 +2581,26 @@ define amdgpu_kernel void @extract_byte1_to_f32(ptr addrspace(1) noalias %out, p
;
; GFX10-LABEL: extract_byte1_to_f32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cvt_f32_ubyte1_e32 v0, v0
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX9-LABEL: extract_byte1_to_f32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX9-NEXT: global_load_dword v0, v0, s[6:7]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte1_e32 v0, v0
-; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: extract_byte1_to_f32:
@@ -2667,26 +2665,26 @@ define amdgpu_kernel void @extract_byte2_to_f32(ptr addrspace(1) noalias %out, p
;
; GFX10-LABEL: extract_byte2_to_f32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cvt_f32_ubyte2_e32 v0, v0
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX9-LABEL: extract_byte2_to_f32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX9-NEXT: global_load_dword v0, v0, s[6:7]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte2_e32 v0, v0
-; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: extract_byte2_to_f32:
@@ -2751,26 +2749,26 @@ define amdgpu_kernel void @extract_byte3_to_f32(ptr addrspace(1) noalias %out, p
;
; GFX10-LABEL: extract_byte3_to_f32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX10-NEXT: global_load_dword v0, v0, s[6:7]
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_cvt_f32_ubyte3_e32 v0, v0
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX9-LABEL: extract_byte3_to_f32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v0, v0, s[2:3]
+; GFX9-NEXT: global_load_dword v0, v0, s[6:7]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte3_e32 v0, v0
-; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: extract_byte3_to_f32:
@@ -2854,16 +2852,16 @@ define amdgpu_kernel void @cvt_ubyte0_or_multiuse(ptr addrspace(1) %in, ptr addr
;
; GFX9-LABEL: cvt_ubyte0_or_multiuse:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v0, v0, s[0:1]
+; GFX9-NEXT: global_load_dword v0, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_e32 v0, 0x80000001, v0
; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v2, v0
; GFX9-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX9-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX9-NEXT: global_store_dword v1, v0, s[6:7]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: cvt_ubyte0_or_multiuse:
diff --git a/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll b/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
index f298a95c63485..6db65752db54f 100644
--- a/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
+++ b/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
@@ -376,22 +376,22 @@ define amdgpu_kernel void @uniform_vec_i16_LH(ptr addrspace(1) %out, i16 %a, i32
;
; GFX9-LABEL: uniform_vec_i16_LH:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_pack_lh_b32_b16 s2, s2, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_pack_lh_b32_b16 s0, s6, s7
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX906-LABEL: uniform_vec_i16_LH:
; GFX906: ; %bb.0:
-; GFX906-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX906-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX906-NEXT: v_mov_b32_e32 v0, 0
; GFX906-NEXT: s_waitcnt lgkmcnt(0)
-; GFX906-NEXT: s_pack_lh_b32_b16 s2, s2, s3
-; GFX906-NEXT: v_mov_b32_e32 v1, s2
-; GFX906-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX906-NEXT: s_pack_lh_b32_b16 s0, s6, s7
+; GFX906-NEXT: v_mov_b32_e32 v1, s0
+; GFX906-NEXT: global_store_dword v0, v1, s[4:5]
; GFX906-NEXT: s_endpgm
;
; GFX11-LABEL: uniform_vec_i16_LH:
@@ -466,22 +466,22 @@ define amdgpu_kernel void @uniform_vec_i16_HH(ptr addrspace(1) %out, i32 %a, i32
;
; GFX9-LABEL: uniform_vec_i16_HH:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_pack_hh_b32_b16 s2, s2, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_pack_hh_b32_b16 s0, s6, s7
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX906-LABEL: uniform_vec_i16_HH:
; GFX906: ; %bb.0:
-; GFX906-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX906-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX906-NEXT: v_mov_b32_e32 v0, 0
; GFX906-NEXT: s_waitcnt lgkmcnt(0)
-; GFX906-NEXT: s_pack_hh_b32_b16 s2, s2, s3
-; GFX906-NEXT: v_mov_b32_e32 v1, s2
-; GFX906-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX906-NEXT: s_pack_hh_b32_b16 s0, s6, s7
+; GFX906-NEXT: v_mov_b32_e32 v1, s0
+; GFX906-NEXT: global_store_dword v0, v1, s[4:5]
; GFX906-NEXT: s_endpgm
;
; GFX11-LABEL: uniform_vec_i16_HH:
diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
index bd483f4c07071..fe672f1b3b131 100644
--- a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
@@ -1842,21 +1842,21 @@ define amdgpu_kernel void @s_copysign_v2f16(ptr addrspace(1) %arg_out, <2 x half
;
; GFX9-LABEL: s_copysign_v2f16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX9-NEXT: s_movk_i32 s4, 0x7fff
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX9-NEXT: s_movk_i32 s0, 0x7fff
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: v_mov_b32_e32 v2, s3
-; GFX9-NEXT: s_lshr_b32 s3, s3, 16
-; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: v_bfi_b32 v1, s4, v1, v2
+; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v2, s7
+; GFX9-NEXT: s_lshr_b32 s1, s7, 16
+; GFX9-NEXT: s_lshr_b32 s2, s6, 16
+; GFX9-NEXT: v_bfi_b32 v1, s0, v1, v2
; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
-; GFX9-NEXT: v_bfi_b32 v2, s4, v2, v3
+; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: v_bfi_b32 v2, s0, v2, v3
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: s_copysign_v2f16:
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv.ll b/llvm/test/CodeGen/AMDGPU/fdiv.ll
index c6b730e3fd5d6..93105e57a5918 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv.ll
@@ -118,10 +118,10 @@ define amdgpu_kernel void @s_fdiv_f32_ninf(ptr addrspace(1) %out, float %a, floa
;
; GFX10-LABEL: s_fdiv_f32_ninf:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_div_scale_f32 v0, s4, s3, s3, s2
-; GFX10-NEXT: v_div_scale_f32 v2, vcc_lo, s2, s3, s2
+; GFX10-NEXT: v_div_scale_f32 v0, s0, s7, s7, s6
+; GFX10-NEXT: v_div_scale_f32 v2, vcc_lo, s6, s7, s6
; GFX10-NEXT: v_rcp_f32_e32 v1, v0
; GFX10-NEXT: s_denorm_mode 15
; GFX10-NEXT: v_fma_f32 v3, -v0, v1, 1.0
@@ -133,8 +133,8 @@ define amdgpu_kernel void @s_fdiv_f32_ninf(ptr addrspace(1) %out, float %a, floa
; GFX10-NEXT: s_denorm_mode 12
; GFX10-NEXT: v_div_fmas_f32 v0, v0, v1, v3
; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_div_fixup_f32 v0, v0, s3, s2
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: v_div_fixup_f32 v0, v0, s7, s6
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_fdiv_f32_ninf:
@@ -275,21 +275,21 @@ define amdgpu_kernel void @s_fdiv_f32_ieee(ptr addrspace(1) %out, float %a, floa
;
; GFX10-LABEL: s_fdiv_f32_ieee:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_div_scale_f32 v0, s4, s3, s3, s2
+; GFX10-NEXT: v_div_scale_f32 v0, s0, s7, s7, s6
; GFX10-NEXT: v_rcp_f32_e32 v1, v0
; GFX10-NEXT: v_fma_f32 v2, -v0, v1, 1.0
; GFX10-NEXT: v_fmac_f32_e32 v1, v2, v1
-; GFX10-NEXT: v_div_scale_f32 v2, vcc_lo, s2, s3, s2
+; GFX10-NEXT: v_div_scale_f32 v2, vcc_lo, s6, s7, s6
; GFX10-NEXT: v_mul_f32_e32 v3, v2, v1
; GFX10-NEXT: v_fma_f32 v4, -v0, v3, v2
; GFX10-NEXT: v_fmac_f32_e32 v3, v4, v1
; GFX10-NEXT: v_fma_f32 v0, -v0, v3, v2
; GFX10-NEXT: v_div_fmas_f32 v0, v0, v1, v3
; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_div_fixup_f32 v0, v0, s3, s2
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: v_div_fixup_f32 v0, v0, s7, s6
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_fdiv_f32_ieee:
@@ -370,16 +370,16 @@ define amdgpu_kernel void @s_fdiv_25ulp_f32(ptr addrspace(1) %out, float %a, flo
;
; GFX10-LABEL: s_fdiv_25ulp_f32:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_cmp_lt_f32_e64 s4, 0x6f800000, |s3|
-; GFX10-NEXT: v_cndmask_b32_e64 v0, 1.0, 0x2f800000, s4
-; GFX10-NEXT: v_mul_f32_e32 v1, s3, v0
+; GFX10-NEXT: v_cmp_lt_f32_e64 s0, 0x6f800000, |s7|
+; GFX10-NEXT: v_cndmask_b32_e64 v0, 1.0, 0x2f800000, s0
+; GFX10-NEXT: v_mul_f32_e32 v1, s7, v0
; GFX10-NEXT: v_rcp_f32_e32 v1, v1
-; GFX10-NEXT: v_mul_f32_e32 v1, s2, v1
+; GFX10-NEXT: v_mul_f32_e32 v1, s6, v1
; GFX10-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX10-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX10-NEXT: global_store_dword v2, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_fdiv_25ulp_f32:
@@ -482,18 +482,18 @@ define amdgpu_kernel void @s_fdiv_25ulp_ieee_f32(ptr addrspace(1) %out, float %a
;
; GFX10-LABEL: s_fdiv_25ulp_ieee_f32:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_frexp_mant_f32_e32 v0, s3
-; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v1, s3
-; GFX10-NEXT: v_frexp_mant_f32_e32 v2, s2
-; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v3, s2
+; GFX10-NEXT: v_frexp_mant_f32_e32 v0, s7
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v1, s7
+; GFX10-NEXT: v_frexp_mant_f32_e32 v2, s6
+; GFX10-NEXT: v_frexp_exp_i32_f32_e32 v3, s6
; GFX10-NEXT: v_rcp_f32_e32 v0, v0
; GFX10-NEXT: v_sub_nc_u32_e32 v1, v3, v1
; GFX10-NEXT: v_mul_f32_e32 v0, v2, v0
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX10-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX10-NEXT: global_store_dword v2, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_fdiv_25ulp_ieee_f32:
@@ -559,12 +559,12 @@ define amdgpu_kernel void @s_fdiv_fast_ieee_f32(ptr addrspace(1) %out, float %a,
;
; GFX10-LABEL: s_fdiv_fast_ieee_f32:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_rcp_f32_e32 v0, s3
-; GFX10-NEXT: v_mul_f32_e32 v0, s2, v0
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: v_rcp_f32_e32 v0, s7
+; GFX10-NEXT: v_mul_f32_e32 v0, s6, v0
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_fdiv_fast_ieee_f32:
@@ -623,12 +623,12 @@ define amdgpu_kernel void @s_fdiv_f32_fast_math(ptr addrspace(1) %out, float %a,
;
; GFX10-LABEL: s_fdiv_f32_fast_math:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_rcp_f32_e32 v0, s3
-; GFX10-NEXT: v_mul_f32_e32 v0, s2, v0
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: v_rcp_f32_e32 v0, s7
+; GFX10-NEXT: v_mul_f32_e32 v0, s6, v0
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_fdiv_f32_fast_math:
@@ -687,12 +687,12 @@ define amdgpu_kernel void @s_fdiv_ulp25_f32_fast_math(ptr addrspace(1) %out, flo
;
; GFX10-LABEL: s_fdiv_ulp25_f32_fast_math:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_rcp_f32_e32 v0, s3
-; GFX10-NEXT: v_mul_f32_e32 v0, s2, v0
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: v_rcp_f32_e32 v0, s7
+; GFX10-NEXT: v_mul_f32_e32 v0, s6, v0
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_fdiv_ulp25_f32_fast_math:
@@ -829,10 +829,10 @@ define amdgpu_kernel void @s_fdiv_f32_arcp_daz(ptr addrspace(1) %out, float %a,
;
; GFX10-LABEL: s_fdiv_f32_arcp_daz:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_div_scale_f32 v0, s4, s3, s3, s2
-; GFX10-NEXT: v_div_scale_f32 v2, vcc_lo, s2, s3, s2
+; GFX10-NEXT: v_div_scale_f32 v0, s0, s7, s7, s6
+; GFX10-NEXT: v_div_scale_f32 v2, vcc_lo, s6, s7, s6
; GFX10-NEXT: v_rcp_f32_e32 v1, v0
; GFX10-NEXT: s_denorm_mode 15
; GFX10-NEXT: v_fma_f32 v3, -v0, v1, 1.0
@@ -844,8 +844,8 @@ define amdgpu_kernel void @s_fdiv_f32_arcp_daz(ptr addrspace(1) %out, float %a,
; GFX10-NEXT: s_denorm_mode 12
; GFX10-NEXT: v_div_fmas_f32 v0, v0, v1, v3
; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_div_fixup_f32 v0, v0, s3, s2
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: v_div_fixup_f32 v0, v0, s7, s6
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_fdiv_f32_arcp_daz:
@@ -916,12 +916,12 @@ define amdgpu_kernel void @s_fdiv_f32_arcp_ninf(ptr addrspace(1) %out, float %a,
;
; GFX10-LABEL: s_fdiv_f32_arcp_ninf:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_rcp_f32_e32 v0, s3
-; GFX10-NEXT: v_mul_f32_e32 v0, s2, v0
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: v_rcp_f32_e32 v0, s7
+; GFX10-NEXT: v_mul_f32_e32 v0, s6, v0
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_fdiv_f32_arcp_ninf:
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics.ll
index c9618d43943ef..8510e26a3eafb 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics.ll
@@ -4315,12 +4315,12 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_offset(ptr %out, i32 %in, i32 %old
;
; GCN3-LABEL: atomic_cmpxchg_i32_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GCN3-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v2, s2
-; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v3, s3
+; GCN3-NEXT: v_mov_b32_e32 v0, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v1, s5
+; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -4570,12 +4570,12 @@ define amdgpu_kernel void @atomic_cmpxchg_i32(ptr %out, i32 %in, i32 %old) {
;
; GCN3-LABEL: atomic_cmpxchg_i32:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GCN3-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v2, s2
-; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v3, s3
+; GCN3-NEXT: v_mov_b32_e32 v0, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v1, s5
+; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll
index 4d80e9124f41f..9c2faf622623d 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll
@@ -3883,13 +3883,13 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr %out, i32 %in, i32 %
;
; GCN3-LABEL: atomic_max_i32_addr64_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GCN3-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_ashr_i32 s5, s3, 31
-; GCN3-NEXT: s_mov_b32 s4, s3
-; GCN3-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; GCN3-NEXT: s_add_u32 s0, s0, s4
-; GCN3-NEXT: s_addc_u32 s1, s1, s5
+; GCN3-NEXT: s_ashr_i32 s1, s7, 31
+; GCN3-NEXT: s_mov_b32 s0, s7
+; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GCN3-NEXT: s_add_u32 s0, s4, s0
+; GCN3-NEXT: s_addc_u32 s1, s5, s1
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
@@ -3897,7 +3897,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr %out, i32 %in, i32 %
; GCN3-NEXT: .LBB88_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_max_i32_e32 v2, s2, v3
+; GCN3-NEXT: v_max_i32_e32 v2, s6, v3
; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -4085,13 +4085,13 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr %out, i32 %in, i32 %index)
;
; GCN3-LABEL: atomic_max_i32_addr64:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GCN3-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_ashr_i32 s5, s3, 31
-; GCN3-NEXT: s_mov_b32 s4, s3
-; GCN3-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; GCN3-NEXT: s_add_u32 s0, s0, s4
-; GCN3-NEXT: s_addc_u32 s1, s1, s5
+; GCN3-NEXT: s_ashr_i32 s1, s7, 31
+; GCN3-NEXT: s_mov_b32 s0, s7
+; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GCN3-NEXT: s_add_u32 s0, s4, s0
+; GCN3-NEXT: s_addc_u32 s1, s5, s1
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
; GCN3-NEXT: flat_load_dword v3, v[0:1]
@@ -4099,7 +4099,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr %out, i32 %in, i32 %index)
; GCN3-NEXT: .LBB90_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_max_i32_e32 v2, s2, v3
+; GCN3-NEXT: v_max_i32_e32 v2, s6, v3
; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -5026,13 +5026,13 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr %out, i32 %in, i32
;
; GCN3-LABEL: atomic_umax_i32_addr64_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GCN3-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_ashr_i32 s5, s3, 31
-; GCN3-NEXT: s_mov_b32 s4, s3
-; GCN3-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; GCN3-NEXT: s_add_u32 s0, s0, s4
-; GCN3-NEXT: s_addc_u32 s1, s1, s5
+; GCN3-NEXT: s_ashr_i32 s1, s7, 31
+; GCN3-NEXT: s_mov_b32 s0, s7
+; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GCN3-NEXT: s_add_u32 s0, s4, s0
+; GCN3-NEXT: s_addc_u32 s1, s5, s1
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
@@ -5040,7 +5040,7 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr %out, i32 %in, i32
; GCN3-NEXT: .LBB102_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_max_u32_e32 v2, s2, v3
+; GCN3-NEXT: v_max_u32_e32 v2, s6, v3
; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -6820,13 +6820,13 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr %out, i32 %in, i32 %
;
; GCN3-LABEL: atomic_min_i32_addr64_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GCN3-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_ashr_i32 s5, s3, 31
-; GCN3-NEXT: s_mov_b32 s4, s3
-; GCN3-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; GCN3-NEXT: s_add_u32 s0, s0, s4
-; GCN3-NEXT: s_addc_u32 s1, s1, s5
+; GCN3-NEXT: s_ashr_i32 s1, s7, 31
+; GCN3-NEXT: s_mov_b32 s0, s7
+; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GCN3-NEXT: s_add_u32 s0, s4, s0
+; GCN3-NEXT: s_addc_u32 s1, s5, s1
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
@@ -6834,7 +6834,7 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr %out, i32 %in, i32 %
; GCN3-NEXT: .LBB125_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_min_i32_e32 v2, s2, v3
+; GCN3-NEXT: v_min_i32_e32 v2, s6, v3
; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
diff --git a/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll b/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll
index 9300dfcb16e8a..70cdfeb6d4954 100644
--- a/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmuladd.f16.ll
@@ -1425,49 +1425,49 @@ define amdgpu_kernel void @mad_sub_f16(ptr addrspace(1) noalias nocapture %out,
;
; GFX10-FLUSH-LABEL: mad_sub_f16:
; GFX10-FLUSH: ; %bb.0:
-; GFX10-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-FLUSH-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-FLUSH-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX10-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-FLUSH-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
+; GFX10-FLUSH-NEXT: global_load_ushort v1, v0, s[6:7] glc dlc
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLUSH-NEXT: global_load_ushort v2, v0, s[2:3] offset:2 glc dlc
+; GFX10-FLUSH-NEXT: global_load_ushort v2, v0, s[6:7] offset:2 glc dlc
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLUSH-NEXT: global_load_ushort v3, v0, s[2:3] offset:4 glc dlc
+; GFX10-FLUSH-NEXT: global_load_ushort v3, v0, s[6:7] offset:4 glc dlc
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLUSH-NEXT: v_mul_f16_e32 v1, v1, v2
; GFX10-FLUSH-NEXT: v_sub_f16_e32 v1, v1, v3
-; GFX10-FLUSH-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-FLUSH-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-FLUSH-NEXT: s_endpgm
;
; GFX10-DENORM-STRICT-LABEL: mad_sub_f16:
; GFX10-DENORM-STRICT: ; %bb.0:
-; GFX10-DENORM-STRICT-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-DENORM-STRICT-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-DENORM-STRICT-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX10-DENORM-STRICT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DENORM-STRICT-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
+; GFX10-DENORM-STRICT-NEXT: global_load_ushort v1, v0, s[6:7] glc dlc
; GFX10-DENORM-STRICT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-STRICT-NEXT: global_load_ushort v2, v0, s[2:3] offset:2 glc dlc
+; GFX10-DENORM-STRICT-NEXT: global_load_ushort v2, v0, s[6:7] offset:2 glc dlc
; GFX10-DENORM-STRICT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-STRICT-NEXT: global_load_ushort v3, v0, s[2:3] offset:4 glc dlc
+; GFX10-DENORM-STRICT-NEXT: global_load_ushort v3, v0, s[6:7] offset:4 glc dlc
; GFX10-DENORM-STRICT-NEXT: s_waitcnt vmcnt(0)
; GFX10-DENORM-STRICT-NEXT: v_mul_f16_e32 v1, v1, v2
; GFX10-DENORM-STRICT-NEXT: v_sub_f16_e32 v1, v1, v3
-; GFX10-DENORM-STRICT-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-DENORM-STRICT-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-DENORM-STRICT-NEXT: s_endpgm
;
; GFX10-DENORM-CONTRACT-LABEL: mad_sub_f16:
; GFX10-DENORM-CONTRACT: ; %bb.0:
-; GFX10-DENORM-CONTRACT-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-DENORM-CONTRACT-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-DENORM-CONTRACT-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
+; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v1, v0, s[6:7] glc dlc
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v2, v0, s[2:3] offset:2 glc dlc
+; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v2, v0, s[6:7] offset:2 glc dlc
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v3, v0, s[2:3] offset:4 glc dlc
+; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v3, v0, s[6:7] offset:4 glc dlc
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
; GFX10-DENORM-CONTRACT-NEXT: v_fma_f16 v1, v1, v2, -v3
-; GFX10-DENORM-CONTRACT-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-DENORM-CONTRACT-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-DENORM-CONTRACT-NEXT: s_endpgm
;
; GFX11-FLUSH-LABEL: mad_sub_f16:
@@ -1600,49 +1600,49 @@ define amdgpu_kernel void @mad_sub_inv_f16(ptr addrspace(1) noalias nocapture %o
;
; GFX10-FLUSH-LABEL: mad_sub_inv_f16:
; GFX10-FLUSH: ; %bb.0:
-; GFX10-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-FLUSH-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-FLUSH-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX10-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-FLUSH-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
+; GFX10-FLUSH-NEXT: global_load_ushort v1, v0, s[6:7] glc dlc
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLUSH-NEXT: global_load_ushort v2, v0, s[2:3] offset:2 glc dlc
+; GFX10-FLUSH-NEXT: global_load_ushort v2, v0, s[6:7] offset:2 glc dlc
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLUSH-NEXT: global_load_ushort v3, v0, s[2:3] offset:4 glc dlc
+; GFX10-FLUSH-NEXT: global_load_ushort v3, v0, s[6:7] offset:4 glc dlc
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLUSH-NEXT: v_mul_f16_e32 v1, v1, v2
; GFX10-FLUSH-NEXT: v_sub_f16_e32 v1, v3, v1
-; GFX10-FLUSH-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-FLUSH-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-FLUSH-NEXT: s_endpgm
;
; GFX10-DENORM-STRICT-LABEL: mad_sub_inv_f16:
; GFX10-DENORM-STRICT: ; %bb.0:
-; GFX10-DENORM-STRICT-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-DENORM-STRICT-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-DENORM-STRICT-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX10-DENORM-STRICT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DENORM-STRICT-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
+; GFX10-DENORM-STRICT-NEXT: global_load_ushort v1, v0, s[6:7] glc dlc
; GFX10-DENORM-STRICT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-STRICT-NEXT: global_load_ushort v2, v0, s[2:3] offset:2 glc dlc
+; GFX10-DENORM-STRICT-NEXT: global_load_ushort v2, v0, s[6:7] offset:2 glc dlc
; GFX10-DENORM-STRICT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-STRICT-NEXT: global_load_ushort v3, v0, s[2:3] offset:4 glc dlc
+; GFX10-DENORM-STRICT-NEXT: global_load_ushort v3, v0, s[6:7] offset:4 glc dlc
; GFX10-DENORM-STRICT-NEXT: s_waitcnt vmcnt(0)
; GFX10-DENORM-STRICT-NEXT: v_mul_f16_e32 v1, v1, v2
; GFX10-DENORM-STRICT-NEXT: v_sub_f16_e32 v1, v3, v1
-; GFX10-DENORM-STRICT-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-DENORM-STRICT-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-DENORM-STRICT-NEXT: s_endpgm
;
; GFX10-DENORM-CONTRACT-LABEL: mad_sub_inv_f16:
; GFX10-DENORM-CONTRACT: ; %bb.0:
-; GFX10-DENORM-CONTRACT-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-DENORM-CONTRACT-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-DENORM-CONTRACT-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
+; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v1, v0, s[6:7] glc dlc
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v2, v0, s[2:3] offset:2 glc dlc
+; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v2, v0, s[6:7] offset:2 glc dlc
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v3, v0, s[2:3] offset:4 glc dlc
+; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v3, v0, s[6:7] offset:4 glc dlc
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
; GFX10-DENORM-CONTRACT-NEXT: v_fma_f16 v1, -v1, v2, v3
-; GFX10-DENORM-CONTRACT-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-DENORM-CONTRACT-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-DENORM-CONTRACT-NEXT: s_endpgm
;
; GFX11-FLUSH-LABEL: mad_sub_inv_f16:
@@ -1775,49 +1775,49 @@ define amdgpu_kernel void @mad_sub_fabs_f16(ptr addrspace(1) noalias nocapture %
;
; GFX10-FLUSH-LABEL: mad_sub_fabs_f16:
; GFX10-FLUSH: ; %bb.0:
-; GFX10-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-FLUSH-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-FLUSH-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX10-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-FLUSH-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
+; GFX10-FLUSH-NEXT: global_load_ushort v1, v0, s[6:7] glc dlc
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLUSH-NEXT: global_load_ushort v2, v0, s[2:3] offset:2 glc dlc
+; GFX10-FLUSH-NEXT: global_load_ushort v2, v0, s[6:7] offset:2 glc dlc
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLUSH-NEXT: global_load_ushort v3, v0, s[2:3] offset:4 glc dlc
+; GFX10-FLUSH-NEXT: global_load_ushort v3, v0, s[6:7] offset:4 glc dlc
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLUSH-NEXT: v_mul_f16_e32 v1, v1, v2
; GFX10-FLUSH-NEXT: v_sub_f16_e64 v1, v1, |v3|
-; GFX10-FLUSH-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-FLUSH-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-FLUSH-NEXT: s_endpgm
;
; GFX10-DENORM-STRICT-LABEL: mad_sub_fabs_f16:
; GFX10-DENORM-STRICT: ; %bb.0:
-; GFX10-DENORM-STRICT-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-DENORM-STRICT-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-DENORM-STRICT-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX10-DENORM-STRICT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DENORM-STRICT-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
+; GFX10-DENORM-STRICT-NEXT: global_load_ushort v1, v0, s[6:7] glc dlc
; GFX10-DENORM-STRICT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-STRICT-NEXT: global_load_ushort v2, v0, s[2:3] offset:2 glc dlc
+; GFX10-DENORM-STRICT-NEXT: global_load_ushort v2, v0, s[6:7] offset:2 glc dlc
; GFX10-DENORM-STRICT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-STRICT-NEXT: global_load_ushort v3, v0, s[2:3] offset:4 glc dlc
+; GFX10-DENORM-STRICT-NEXT: global_load_ushort v3, v0, s[6:7] offset:4 glc dlc
; GFX10-DENORM-STRICT-NEXT: s_waitcnt vmcnt(0)
; GFX10-DENORM-STRICT-NEXT: v_mul_f16_e32 v1, v1, v2
; GFX10-DENORM-STRICT-NEXT: v_sub_f16_e64 v1, v1, |v3|
-; GFX10-DENORM-STRICT-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-DENORM-STRICT-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-DENORM-STRICT-NEXT: s_endpgm
;
; GFX10-DENORM-CONTRACT-LABEL: mad_sub_fabs_f16:
; GFX10-DENORM-CONTRACT: ; %bb.0:
-; GFX10-DENORM-CONTRACT-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-DENORM-CONTRACT-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-DENORM-CONTRACT-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
+; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v1, v0, s[6:7] glc dlc
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v2, v0, s[2:3] offset:2 glc dlc
+; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v2, v0, s[6:7] offset:2 glc dlc
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v3, v0, s[2:3] offset:4 glc dlc
+; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v3, v0, s[6:7] offset:4 glc dlc
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
; GFX10-DENORM-CONTRACT-NEXT: v_fma_f16 v1, v1, v2, -|v3|
-; GFX10-DENORM-CONTRACT-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-DENORM-CONTRACT-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-DENORM-CONTRACT-NEXT: s_endpgm
;
; GFX11-FLUSH-LABEL: mad_sub_fabs_f16:
@@ -1951,49 +1951,49 @@ define amdgpu_kernel void @mad_sub_fabs_inv_f16(ptr addrspace(1) noalias nocaptu
;
; GFX10-FLUSH-LABEL: mad_sub_fabs_inv_f16:
; GFX10-FLUSH: ; %bb.0:
-; GFX10-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-FLUSH-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-FLUSH-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX10-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-FLUSH-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
+; GFX10-FLUSH-NEXT: global_load_ushort v1, v0, s[6:7] glc dlc
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLUSH-NEXT: global_load_ushort v2, v0, s[2:3] offset:2 glc dlc
+; GFX10-FLUSH-NEXT: global_load_ushort v2, v0, s[6:7] offset:2 glc dlc
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLUSH-NEXT: global_load_ushort v3, v0, s[2:3] offset:4 glc dlc
+; GFX10-FLUSH-NEXT: global_load_ushort v3, v0, s[6:7] offset:4 glc dlc
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLUSH-NEXT: v_mul_f16_e32 v1, v1, v2
; GFX10-FLUSH-NEXT: v_sub_f16_e64 v1, |v3|, v1
-; GFX10-FLUSH-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-FLUSH-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-FLUSH-NEXT: s_endpgm
;
; GFX10-DENORM-STRICT-LABEL: mad_sub_fabs_inv_f16:
; GFX10-DENORM-STRICT: ; %bb.0:
-; GFX10-DENORM-STRICT-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-DENORM-STRICT-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-DENORM-STRICT-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX10-DENORM-STRICT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DENORM-STRICT-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
+; GFX10-DENORM-STRICT-NEXT: global_load_ushort v1, v0, s[6:7] glc dlc
; GFX10-DENORM-STRICT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-STRICT-NEXT: global_load_ushort v2, v0, s[2:3] offset:2 glc dlc
+; GFX10-DENORM-STRICT-NEXT: global_load_ushort v2, v0, s[6:7] offset:2 glc dlc
; GFX10-DENORM-STRICT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-STRICT-NEXT: global_load_ushort v3, v0, s[2:3] offset:4 glc dlc
+; GFX10-DENORM-STRICT-NEXT: global_load_ushort v3, v0, s[6:7] offset:4 glc dlc
; GFX10-DENORM-STRICT-NEXT: s_waitcnt vmcnt(0)
; GFX10-DENORM-STRICT-NEXT: v_mul_f16_e32 v1, v1, v2
; GFX10-DENORM-STRICT-NEXT: v_sub_f16_e64 v1, |v3|, v1
-; GFX10-DENORM-STRICT-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-DENORM-STRICT-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-DENORM-STRICT-NEXT: s_endpgm
;
; GFX10-DENORM-CONTRACT-LABEL: mad_sub_fabs_inv_f16:
; GFX10-DENORM-CONTRACT: ; %bb.0:
-; GFX10-DENORM-CONTRACT-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-DENORM-CONTRACT-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-DENORM-CONTRACT-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
+; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v1, v0, s[6:7] glc dlc
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v2, v0, s[2:3] offset:2 glc dlc
+; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v2, v0, s[6:7] offset:2 glc dlc
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v3, v0, s[2:3] offset:4 glc dlc
+; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v3, v0, s[6:7] offset:4 glc dlc
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
; GFX10-DENORM-CONTRACT-NEXT: v_fma_f16 v1, -v1, v2, |v3|
-; GFX10-DENORM-CONTRACT-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-DENORM-CONTRACT-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-DENORM-CONTRACT-NEXT: s_endpgm
;
; GFX11-FLUSH-LABEL: mad_sub_fabs_inv_f16:
@@ -2127,49 +2127,49 @@ define amdgpu_kernel void @neg_neg_mad_f16(ptr addrspace(1) noalias nocapture %o
;
; GFX10-FLUSH-LABEL: neg_neg_mad_f16:
; GFX10-FLUSH: ; %bb.0:
-; GFX10-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-FLUSH-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-FLUSH-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX10-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-FLUSH-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
+; GFX10-FLUSH-NEXT: global_load_ushort v1, v0, s[6:7] glc dlc
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLUSH-NEXT: global_load_ushort v2, v0, s[2:3] offset:2 glc dlc
+; GFX10-FLUSH-NEXT: global_load_ushort v2, v0, s[6:7] offset:2 glc dlc
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLUSH-NEXT: global_load_ushort v3, v0, s[2:3] offset:4 glc dlc
+; GFX10-FLUSH-NEXT: global_load_ushort v3, v0, s[6:7] offset:4 glc dlc
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLUSH-NEXT: v_mul_f16_e32 v1, v1, v2
; GFX10-FLUSH-NEXT: v_add_f16_e32 v1, v3, v1
-; GFX10-FLUSH-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-FLUSH-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-FLUSH-NEXT: s_endpgm
;
; GFX10-DENORM-STRICT-LABEL: neg_neg_mad_f16:
; GFX10-DENORM-STRICT: ; %bb.0:
-; GFX10-DENORM-STRICT-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-DENORM-STRICT-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-DENORM-STRICT-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX10-DENORM-STRICT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DENORM-STRICT-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
+; GFX10-DENORM-STRICT-NEXT: global_load_ushort v1, v0, s[6:7] glc dlc
; GFX10-DENORM-STRICT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-STRICT-NEXT: global_load_ushort v2, v0, s[2:3] offset:2 glc dlc
+; GFX10-DENORM-STRICT-NEXT: global_load_ushort v2, v0, s[6:7] offset:2 glc dlc
; GFX10-DENORM-STRICT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-STRICT-NEXT: global_load_ushort v3, v0, s[2:3] offset:4 glc dlc
+; GFX10-DENORM-STRICT-NEXT: global_load_ushort v3, v0, s[6:7] offset:4 glc dlc
; GFX10-DENORM-STRICT-NEXT: s_waitcnt vmcnt(0)
; GFX10-DENORM-STRICT-NEXT: v_mul_f16_e32 v1, v1, v2
; GFX10-DENORM-STRICT-NEXT: v_add_f16_e32 v1, v3, v1
-; GFX10-DENORM-STRICT-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-DENORM-STRICT-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-DENORM-STRICT-NEXT: s_endpgm
;
; GFX10-DENORM-CONTRACT-LABEL: neg_neg_mad_f16:
; GFX10-DENORM-CONTRACT: ; %bb.0:
-; GFX10-DENORM-CONTRACT-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-DENORM-CONTRACT-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-DENORM-CONTRACT-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
+; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v1, v0, s[6:7] glc dlc
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v2, v0, s[2:3] offset:2 glc dlc
+; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v2, v0, s[6:7] offset:2 glc dlc
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v3, v0, s[2:3] offset:4 glc dlc
+; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v3, v0, s[6:7] offset:4 glc dlc
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
; GFX10-DENORM-CONTRACT-NEXT: v_fmac_f16_e32 v3, v1, v2
-; GFX10-DENORM-CONTRACT-NEXT: global_store_short v0, v3, s[0:1]
+; GFX10-DENORM-CONTRACT-NEXT: global_store_short v0, v3, s[4:5]
; GFX10-DENORM-CONTRACT-NEXT: s_endpgm
;
; GFX11-FLUSH-LABEL: neg_neg_mad_f16:
@@ -2304,49 +2304,49 @@ define amdgpu_kernel void @mad_fabs_sub_f16(ptr addrspace(1) noalias nocapture %
;
; GFX10-FLUSH-LABEL: mad_fabs_sub_f16:
; GFX10-FLUSH: ; %bb.0:
-; GFX10-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-FLUSH-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-FLUSH-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX10-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-FLUSH-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
+; GFX10-FLUSH-NEXT: global_load_ushort v1, v0, s[6:7] glc dlc
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLUSH-NEXT: global_load_ushort v2, v0, s[2:3] offset:2 glc dlc
+; GFX10-FLUSH-NEXT: global_load_ushort v2, v0, s[6:7] offset:2 glc dlc
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FLUSH-NEXT: global_load_ushort v3, v0, s[2:3] offset:4 glc dlc
+; GFX10-FLUSH-NEXT: global_load_ushort v3, v0, s[6:7] offset:4 glc dlc
; GFX10-FLUSH-NEXT: s_waitcnt vmcnt(0)
; GFX10-FLUSH-NEXT: v_mul_f16_e64 v1, v1, |v2|
; GFX10-FLUSH-NEXT: v_sub_f16_e32 v1, v1, v3
-; GFX10-FLUSH-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-FLUSH-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-FLUSH-NEXT: s_endpgm
;
; GFX10-DENORM-STRICT-LABEL: mad_fabs_sub_f16:
; GFX10-DENORM-STRICT: ; %bb.0:
-; GFX10-DENORM-STRICT-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-DENORM-STRICT-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-DENORM-STRICT-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX10-DENORM-STRICT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DENORM-STRICT-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
+; GFX10-DENORM-STRICT-NEXT: global_load_ushort v1, v0, s[6:7] glc dlc
; GFX10-DENORM-STRICT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-STRICT-NEXT: global_load_ushort v2, v0, s[2:3] offset:2 glc dlc
+; GFX10-DENORM-STRICT-NEXT: global_load_ushort v2, v0, s[6:7] offset:2 glc dlc
; GFX10-DENORM-STRICT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-STRICT-NEXT: global_load_ushort v3, v0, s[2:3] offset:4 glc dlc
+; GFX10-DENORM-STRICT-NEXT: global_load_ushort v3, v0, s[6:7] offset:4 glc dlc
; GFX10-DENORM-STRICT-NEXT: s_waitcnt vmcnt(0)
; GFX10-DENORM-STRICT-NEXT: v_mul_f16_e64 v1, v1, |v2|
; GFX10-DENORM-STRICT-NEXT: v_sub_f16_e32 v1, v1, v3
-; GFX10-DENORM-STRICT-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-DENORM-STRICT-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-DENORM-STRICT-NEXT: s_endpgm
;
; GFX10-DENORM-CONTRACT-LABEL: mad_fabs_sub_f16:
; GFX10-DENORM-CONTRACT: ; %bb.0:
-; GFX10-DENORM-CONTRACT-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-DENORM-CONTRACT-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-DENORM-CONTRACT-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v1, v0, s[2:3] glc dlc
+; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v1, v0, s[6:7] glc dlc
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v2, v0, s[2:3] offset:2 glc dlc
+; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v2, v0, s[6:7] offset:2 glc dlc
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
-; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v3, v0, s[2:3] offset:4 glc dlc
+; GFX10-DENORM-CONTRACT-NEXT: global_load_ushort v3, v0, s[6:7] offset:4 glc dlc
; GFX10-DENORM-CONTRACT-NEXT: s_waitcnt vmcnt(0)
; GFX10-DENORM-CONTRACT-NEXT: v_fma_f16 v1, v1, |v2|, -v3
-; GFX10-DENORM-CONTRACT-NEXT: global_store_short v0, v1, s[0:1]
+; GFX10-DENORM-CONTRACT-NEXT: global_store_short v0, v1, s[4:5]
; GFX10-DENORM-CONTRACT-NEXT: s_endpgm
;
; GFX11-FLUSH-LABEL: mad_fabs_sub_f16:
diff --git a/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-atomics.ll b/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-atomics.ll
index 105d9246880a4..742aeb96fcc2c 100644
--- a/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-atomics.ll
@@ -271,12 +271,12 @@ define amdgpu_kernel void @raw_buffer_atomic_min_rtn_f32_off4_slc(<4 x i32> inre
;
; GFX10-LABEL: raw_buffer_atomic_min_rtn_f32_off4_slc:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s4
-; GFX10-NEXT: v_mov_b32_e32 v1, s5
-; GFX10-NEXT: buffer_atomic_fmin v0, v1, s[0:3], 4 offen glc slc
-; GFX10-NEXT: v_mov_b32_e32 v1, s6
+; GFX10-NEXT: v_mov_b32_e32 v0, s8
+; GFX10-NEXT: v_mov_b32_e32 v1, s9
+; GFX10-NEXT: buffer_atomic_fmin v0, v1, s[4:7], 4 offen glc slc
+; GFX10-NEXT: v_mov_b32_e32 v1, s10
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ds_write_b32 v1, v0
; GFX10-NEXT: s_endpgm
@@ -657,14 +657,14 @@ define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f32_off4_slc(<4 x i32> inre
;
; GFX10-LABEL: raw_buffer_atomic_max_rtn_f32_off4_slc:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s4
-; GFX10-NEXT: v_mov_b32_e32 v1, s5
-; GFX10-NEXT: buffer_atomic_fmax v0, v1, s[0:3], 4 offen glc slc
+; GFX10-NEXT: v_mov_b32_e32 v0, s8
+; GFX10-NEXT: v_mov_b32_e32 v1, s9
+; GFX10-NEXT: buffer_atomic_fmax v0, v1, s[4:7], 4 offen glc slc
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: global_store_dword v1, v0, s[6:7]
+; GFX10-NEXT: global_store_dword v1, v0, s[10:11]
; GFX10-NEXT: s_endpgm
;
; GFX1030-LABEL: raw_buffer_atomic_max_rtn_f32_off4_slc:
@@ -736,14 +736,14 @@ define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f32_off4_slc(<4 x i32> inre
;
; G_GFX10-LABEL: raw_buffer_atomic_max_rtn_f32_off4_slc:
; G_GFX10: ; %bb.0: ; %main_body
-; G_GFX10-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
+; G_GFX10-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; G_GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; G_GFX10-NEXT: v_mov_b32_e32 v0, s4
-; G_GFX10-NEXT: v_mov_b32_e32 v1, s5
-; G_GFX10-NEXT: buffer_atomic_fmax v0, v1, s[0:3], 4 offen glc slc
+; G_GFX10-NEXT: v_mov_b32_e32 v0, s8
+; G_GFX10-NEXT: v_mov_b32_e32 v1, s9
+; G_GFX10-NEXT: buffer_atomic_fmax v0, v1, s[4:7], 4 offen glc slc
; G_GFX10-NEXT: v_mov_b32_e32 v1, 0
; G_GFX10-NEXT: s_waitcnt vmcnt(0)
-; G_GFX10-NEXT: global_store_dword v1, v0, s[6:7]
+; G_GFX10-NEXT: global_store_dword v1, v0, s[10:11]
; G_GFX10-NEXT: s_endpgm
;
; G_GFX1030-LABEL: raw_buffer_atomic_max_rtn_f32_off4_slc:
diff --git a/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-ptr-atomics.ll b/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-ptr-atomics.ll
index e124aadf4e8c2..950d228f29929 100644
--- a/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-ptr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp-min-max-buffer-ptr-atomics.ll
@@ -248,12 +248,12 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_min_rtn_f32_off4_slc(ptr addrsp
;
; GFX10-LABEL: raw_ptr_buffer_atomic_min_rtn_f32_off4_slc:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s4
-; GFX10-NEXT: v_mov_b32_e32 v1, s5
-; GFX10-NEXT: buffer_atomic_fmin v0, v1, s[0:3], 4 offen glc slc
-; GFX10-NEXT: v_mov_b32_e32 v1, s6
+; GFX10-NEXT: v_mov_b32_e32 v0, s8
+; GFX10-NEXT: v_mov_b32_e32 v1, s9
+; GFX10-NEXT: buffer_atomic_fmin v0, v1, s[4:7], 4 offen glc slc
+; GFX10-NEXT: v_mov_b32_e32 v1, s10
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ds_write_b32 v1, v0
; GFX10-NEXT: s_endpgm
@@ -600,14 +600,14 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f32_off4_slc(ptr addrsp
;
; GFX10-LABEL: raw_ptr_buffer_atomic_max_rtn_f32_off4_slc:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s4
-; GFX10-NEXT: v_mov_b32_e32 v1, s5
-; GFX10-NEXT: buffer_atomic_fmax v0, v1, s[0:3], 4 offen glc slc
+; GFX10-NEXT: v_mov_b32_e32 v0, s8
+; GFX10-NEXT: v_mov_b32_e32 v1, s9
+; GFX10-NEXT: buffer_atomic_fmax v0, v1, s[4:7], 4 offen glc slc
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: global_store_dword v1, v0, s[6:7]
+; GFX10-NEXT: global_store_dword v1, v0, s[10:11]
; GFX10-NEXT: s_endpgm
;
; GFX1030-LABEL: raw_ptr_buffer_atomic_max_rtn_f32_off4_slc:
@@ -665,14 +665,14 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f32_off4_slc(ptr addrsp
;
; G_GFX10-LABEL: raw_ptr_buffer_atomic_max_rtn_f32_off4_slc:
; G_GFX10: ; %bb.0: ; %main_body
-; G_GFX10-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
+; G_GFX10-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; G_GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; G_GFX10-NEXT: v_mov_b32_e32 v0, s4
-; G_GFX10-NEXT: v_mov_b32_e32 v1, s5
-; G_GFX10-NEXT: buffer_atomic_fmax v0, v1, s[0:3], 4 offen glc slc
+; G_GFX10-NEXT: v_mov_b32_e32 v0, s8
+; G_GFX10-NEXT: v_mov_b32_e32 v1, s9
+; G_GFX10-NEXT: buffer_atomic_fmax v0, v1, s[4:7], 4 offen glc slc
; G_GFX10-NEXT: v_mov_b32_e32 v1, 0
; G_GFX10-NEXT: s_waitcnt vmcnt(0)
-; G_GFX10-NEXT: global_store_dword v1, v0, s[6:7]
+; G_GFX10-NEXT: global_store_dword v1, v0, s[10:11]
; G_GFX10-NEXT: s_endpgm
;
; G_GFX1030-LABEL: raw_ptr_buffer_atomic_max_rtn_f32_off4_slc:
diff --git a/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-atomics.ll b/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-atomics.ll
index f18f5752269e0..90c1759070a59 100644
--- a/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-atomics.ll
@@ -452,13 +452,13 @@ define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> inre
;
; GFX10-LABEL: raw_buffer_atomic_max_rtn_f64_off4_slc:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s4
-; GFX10-NEXT: v_mov_b32_e32 v1, s5
-; GFX10-NEXT: v_mov_b32_e32 v2, s6
-; GFX10-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[0:3], 4 offen glc slc
-; GFX10-NEXT: v_mov_b32_e32 v2, s7
+; GFX10-NEXT: v_mov_b32_e32 v0, s8
+; GFX10-NEXT: v_mov_b32_e32 v1, s9
+; GFX10-NEXT: v_mov_b32_e32 v2, s10
+; GFX10-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 4 offen glc slc
+; GFX10-NEXT: v_mov_b32_e32 v2, s11
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ds_write_b64 v2, v[0:1]
; GFX10-NEXT: s_endpgm
@@ -506,13 +506,13 @@ define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> inre
;
; G_GFX10-LABEL: raw_buffer_atomic_max_rtn_f64_off4_slc:
; G_GFX10: ; %bb.0: ; %main_body
-; G_GFX10-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
+; G_GFX10-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; G_GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; G_GFX10-NEXT: v_mov_b32_e32 v0, s4
-; G_GFX10-NEXT: v_mov_b32_e32 v1, s5
-; G_GFX10-NEXT: v_mov_b32_e32 v2, s6
-; G_GFX10-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[0:3], 4 offen glc slc
-; G_GFX10-NEXT: v_mov_b32_e32 v2, s7
+; G_GFX10-NEXT: v_mov_b32_e32 v0, s8
+; G_GFX10-NEXT: v_mov_b32_e32 v1, s9
+; G_GFX10-NEXT: v_mov_b32_e32 v2, s10
+; G_GFX10-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 4 offen glc slc
+; G_GFX10-NEXT: v_mov_b32_e32 v2, s11
; G_GFX10-NEXT: s_waitcnt vmcnt(0)
; G_GFX10-NEXT: ds_write_b64 v2, v[0:1]
; G_GFX10-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-ptr-atomics.ll b/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-ptr-atomics.ll
index 6a2a8c3ce595d..ff6700d10ff53 100644
--- a/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-ptr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-ptr-atomics.ll
@@ -452,13 +452,13 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrsp
;
; GFX10-LABEL: raw_ptr_buffer_atomic_max_rtn_f64_off4_slc:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s4
-; GFX10-NEXT: v_mov_b32_e32 v1, s5
-; GFX10-NEXT: v_mov_b32_e32 v2, s6
-; GFX10-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[0:3], 4 offen glc slc
-; GFX10-NEXT: v_mov_b32_e32 v2, s7
+; GFX10-NEXT: v_mov_b32_e32 v0, s8
+; GFX10-NEXT: v_mov_b32_e32 v1, s9
+; GFX10-NEXT: v_mov_b32_e32 v2, s10
+; GFX10-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 4 offen glc slc
+; GFX10-NEXT: v_mov_b32_e32 v2, s11
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ds_write_b64 v2, v[0:1]
; GFX10-NEXT: s_endpgm
@@ -506,13 +506,13 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrsp
;
; G_GFX10-LABEL: raw_ptr_buffer_atomic_max_rtn_f64_off4_slc:
; G_GFX10: ; %bb.0: ; %main_body
-; G_GFX10-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
+; G_GFX10-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; G_GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; G_GFX10-NEXT: v_mov_b32_e32 v0, s4
-; G_GFX10-NEXT: v_mov_b32_e32 v1, s5
-; G_GFX10-NEXT: v_mov_b32_e32 v2, s6
-; G_GFX10-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[0:3], 4 offen glc slc
-; G_GFX10-NEXT: v_mov_b32_e32 v2, s7
+; G_GFX10-NEXT: v_mov_b32_e32 v0, s8
+; G_GFX10-NEXT: v_mov_b32_e32 v1, s9
+; G_GFX10-NEXT: v_mov_b32_e32 v2, s10
+; G_GFX10-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[4:7], 4 offen glc slc
+; G_GFX10-NEXT: v_mov_b32_e32 v2, s11
; G_GFX10-NEXT: s_waitcnt vmcnt(0)
; G_GFX10-NEXT: ds_write_b64 v2, v[0:1]
; G_GFX10-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/fshl.ll b/llvm/test/CodeGen/AMDGPU/fshl.ll
index ea588df86b846..4f230140f7ba4 100644
--- a/llvm/test/CodeGen/AMDGPU/fshl.ll
+++ b/llvm/test/CodeGen/AMDGPU/fshl.ll
@@ -135,12 +135,12 @@ define amdgpu_kernel void @fshl_i32_imm(ptr addrspace(1) %in, i32 %x, i32 %y) {
;
; GFX9-LABEL: fshl_i32_imm:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: v_alignbit_b32 v1, s2, v1, 25
-; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: v_alignbit_b32 v1, s6, v1, 25
+; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: s_endpgm
;
; R600-LABEL: fshl_i32_imm:
@@ -157,11 +157,11 @@ define amdgpu_kernel void @fshl_i32_imm(ptr addrspace(1) %in, i32 %x, i32 %y) {
;
; GFX10-LABEL: fshl_i32_imm:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_alignbit_b32 v1, s2, s3, 25
-; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: v_alignbit_b32 v1, s6, s7, 25
+; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: fshl_i32_imm:
@@ -732,15 +732,15 @@ define amdgpu_kernel void @orxor2or1(ptr addrspace(1) %in, i32 %a, i32 %b) {
;
; GFX9-LABEL: orxor2or1:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b32 s4, s2, 7
-; GFX9-NEXT: s_or_b32 s4, s3, s4
-; GFX9-NEXT: s_cmp_eq_u32 s4, 0
-; GFX9-NEXT: s_cselect_b32 s2, s2, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_lshl_b32 s0, s6, 7
+; GFX9-NEXT: s_or_b32 s0, s7, s0
+; GFX9-NEXT: s_cmp_eq_u32 s0, 0
+; GFX9-NEXT: s_cselect_b32 s0, s6, s7
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: s_endpgm
;
; R600-LABEL: orxor2or1:
@@ -759,15 +759,15 @@ define amdgpu_kernel void @orxor2or1(ptr addrspace(1) %in, i32 %a, i32 %b) {
;
; GFX10-LABEL: orxor2or1:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_lshl_b32 s4, s2, 7
-; GFX10-NEXT: s_or_b32 s4, s3, s4
-; GFX10-NEXT: s_cmp_eq_u32 s4, 0
-; GFX10-NEXT: s_cselect_b32 s2, s2, s3
-; GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_lshl_b32 s0, s6, 7
+; GFX10-NEXT: s_or_b32 s0, s7, s0
+; GFX10-NEXT: s_cmp_eq_u32 s0, 0
+; GFX10-NEXT: s_cselect_b32 s0, s6, s7
+; GFX10-NEXT: v_mov_b32_e32 v1, s0
+; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: orxor2or1:
diff --git a/llvm/test/CodeGen/AMDGPU/fshr.ll b/llvm/test/CodeGen/AMDGPU/fshr.ll
index dbcebe6e07e3f..31f574d44ab8c 100644
--- a/llvm/test/CodeGen/AMDGPU/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/fshr.ll
@@ -127,12 +127,12 @@ define amdgpu_kernel void @fshr_i32_imm(ptr addrspace(1) %in, i32 %x, i32 %y) {
;
; GFX9-LABEL: fshr_i32_imm:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: v_alignbit_b32 v1, s2, v1, 7
-; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: v_alignbit_b32 v1, s6, v1, 7
+; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: s_endpgm
;
; R600-LABEL: fshr_i32_imm:
@@ -149,11 +149,11 @@ define amdgpu_kernel void @fshr_i32_imm(ptr addrspace(1) %in, i32 %x, i32 %y) {
;
; GFX10-LABEL: fshr_i32_imm:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_alignbit_b32 v1, s2, s3, 7
-; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: v_alignbit_b32 v1, s6, s7, 7
+; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: fshr_i32_imm:
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics.ll b/llvm/test/CodeGen/AMDGPU/global_atomics.ll
index 9bee539b1e4e5..fe6d467bb6281 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics.ll
@@ -4155,12 +4155,12 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_offset(ptr addrspace(1) %out, i32
;
; GFX9-LABEL: atomic_cmpxchg_i32_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
+; GFX9-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: global_atomic_cmpswap v2, v[0:1], s[4:5] offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm
@@ -4406,12 +4406,12 @@ define amdgpu_kernel void @atomic_cmpxchg_i32(ptr addrspace(1) %out, i32 %in, i3
;
; GFX9-LABEL: atomic_cmpxchg_i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: global_atomic_cmpswap v2, v[0:1], s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
index a7ba8a084272b..c7fa2a2ede388 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
@@ -4679,28 +4679,28 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i
;
; GFX9-LABEL: atomic_max_i32_addr64_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_ashr_i32 s5, s3, 31
-; GFX9-NEXT: s_mov_b32 s4, s3
-; GFX9-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; GFX9-NEXT: s_add_u32 s0, s0, s4
-; GFX9-NEXT: s_addc_u32 s1, s1, s5
-; GFX9-NEXT: s_load_dword s3, s[0:1], 0x10
-; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_ashr_i32 s1, s7, 31
+; GFX9-NEXT: s_mov_b32 s0, s7
+; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GFX9-NEXT: s_add_u32 s0, s4, s0
+; GFX9-NEXT: s_addc_u32 s1, s5, s1
+; GFX9-NEXT: s_load_dword s4, s[0:1], 0x10
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: .LBB91_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_max_i32_e32 v0, s2, v1
+; GFX9-NEXT: v_max_i32_e32 v0, s6, v1
; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-NEXT: s_cbranch_execnz .LBB91_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_endpgm
@@ -4890,28 +4890,28 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in,
;
; GFX9-LABEL: atomic_max_i32_addr64:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_ashr_i32 s5, s3, 31
-; GFX9-NEXT: s_mov_b32 s4, s3
-; GFX9-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; GFX9-NEXT: s_add_u32 s0, s0, s4
-; GFX9-NEXT: s_addc_u32 s1, s1, s5
-; GFX9-NEXT: s_load_dword s3, s[0:1], 0x0
-; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_ashr_i32 s1, s7, 31
+; GFX9-NEXT: s_mov_b32 s0, s7
+; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GFX9-NEXT: s_add_u32 s0, s4, s0
+; GFX9-NEXT: s_addc_u32 s1, s5, s1
+; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: .LBB93_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_max_i32_e32 v0, s2, v1
+; GFX9-NEXT: v_max_i32_e32 v0, s6, v1
; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-NEXT: s_cbranch_execnz .LBB93_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_endpgm
@@ -5932,28 +5932,28 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out,
;
; GFX9-LABEL: atomic_umax_i32_addr64_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_ashr_i32 s5, s3, 31
-; GFX9-NEXT: s_mov_b32 s4, s3
-; GFX9-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; GFX9-NEXT: s_add_u32 s0, s0, s4
-; GFX9-NEXT: s_addc_u32 s1, s1, s5
-; GFX9-NEXT: s_load_dword s3, s[0:1], 0x10
-; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_ashr_i32 s1, s7, 31
+; GFX9-NEXT: s_mov_b32 s0, s7
+; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GFX9-NEXT: s_add_u32 s0, s4, s0
+; GFX9-NEXT: s_addc_u32 s1, s5, s1
+; GFX9-NEXT: s_load_dword s4, s[0:1], 0x10
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: .LBB105_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_max_u32_e32 v0, s2, v1
+; GFX9-NEXT: v_max_u32_e32 v0, s6, v1
; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-NEXT: s_cbranch_execnz .LBB105_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_endpgm
@@ -7923,28 +7923,28 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i
;
; GFX9-LABEL: atomic_min_i32_addr64_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_ashr_i32 s5, s3, 31
-; GFX9-NEXT: s_mov_b32 s4, s3
-; GFX9-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; GFX9-NEXT: s_add_u32 s0, s0, s4
-; GFX9-NEXT: s_addc_u32 s1, s1, s5
-; GFX9-NEXT: s_load_dword s3, s[0:1], 0x10
-; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: s_ashr_i32 s1, s7, 31
+; GFX9-NEXT: s_mov_b32 s0, s7
+; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GFX9-NEXT: s_add_u32 s0, s4, s0
+; GFX9-NEXT: s_addc_u32 s1, s5, s1
+; GFX9-NEXT: s_load_dword s4, s[0:1], 0x10
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: .LBB128_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_min_i32_e32 v0, s2, v1
+; GFX9-NEXT: v_min_i32_e32 v0, s6, v1
; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-NEXT: s_cbranch_execnz .LBB128_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll
index b9dc27cb7e019..17fe1721f73d7 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll
@@ -438,121 +438,121 @@ entry:
define amdgpu_kernel void @udiv_i32(ptr addrspace(1) %out, i32 %x, i32 %y) {
; GFX9-LABEL: udiv_i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3
-; GFX9-NEXT: s_sub_i32 s4, 0, s3
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7
+; GFX9-NEXT: s_sub_i32 s0, 0, s7
; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: v_readfirstlane_b32 s5, v0
-; GFX9-NEXT: s_mul_i32 s4, s4, s5
-; GFX9-NEXT: s_mul_hi_u32 s4, s5, s4
-; GFX9-NEXT: s_add_i32 s5, s5, s4
-; GFX9-NEXT: s_mul_hi_u32 s4, s2, s5
-; GFX9-NEXT: s_mul_i32 s5, s4, s3
-; GFX9-NEXT: s_sub_i32 s2, s2, s5
-; GFX9-NEXT: s_add_i32 s6, s4, 1
-; GFX9-NEXT: s_sub_i32 s5, s2, s3
-; GFX9-NEXT: s_cmp_ge_u32 s2, s3
-; GFX9-NEXT: s_cselect_b32 s4, s6, s4
-; GFX9-NEXT: s_cselect_b32 s2, s5, s2
-; GFX9-NEXT: s_add_i32 s5, s4, 1
-; GFX9-NEXT: s_cmp_ge_u32 s2, s3
-; GFX9-NEXT: s_cselect_b32 s2, s5, s4
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_mul_i32 s0, s0, s1
+; GFX9-NEXT: s_mul_hi_u32 s0, s1, s0
+; GFX9-NEXT: s_add_i32 s1, s1, s0
+; GFX9-NEXT: s_mul_hi_u32 s0, s6, s1
+; GFX9-NEXT: s_mul_i32 s1, s0, s7
+; GFX9-NEXT: s_sub_i32 s1, s6, s1
+; GFX9-NEXT: s_add_i32 s2, s0, 1
+; GFX9-NEXT: s_sub_i32 s3, s1, s7
+; GFX9-NEXT: s_cmp_ge_u32 s1, s7
+; GFX9-NEXT: s_cselect_b32 s0, s2, s0
+; GFX9-NEXT: s_cselect_b32 s1, s3, s1
+; GFX9-NEXT: s_add_i32 s2, s0, 1
+; GFX9-NEXT: s_cmp_ge_u32 s1, s7
+; GFX9-NEXT: s_cselect_b32 s0, s2, s0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_endpgm
;
; GFX90A-LABEL: udiv_i32:
; GFX90A: ; %bb.0:
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX90A-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, 0
-; GFX90A-NEXT: v_cvt_f32_u32_e32 v0, s3
-; GFX90A-NEXT: s_sub_i32 s4, 0, s3
+; GFX90A-NEXT: v_cvt_f32_u32_e32 v0, s7
+; GFX90A-NEXT: s_sub_i32 s0, 0, s7
; GFX90A-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX90A-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX90A-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX90A-NEXT: v_readfirstlane_b32 s5, v0
-; GFX90A-NEXT: s_mul_i32 s4, s4, s5
-; GFX90A-NEXT: s_mul_hi_u32 s4, s5, s4
-; GFX90A-NEXT: s_add_i32 s5, s5, s4
-; GFX90A-NEXT: s_mul_hi_u32 s4, s2, s5
-; GFX90A-NEXT: s_mul_i32 s5, s4, s3
-; GFX90A-NEXT: s_sub_i32 s2, s2, s5
-; GFX90A-NEXT: s_add_i32 s6, s4, 1
-; GFX90A-NEXT: s_sub_i32 s5, s2, s3
-; GFX90A-NEXT: s_cmp_ge_u32 s2, s3
-; GFX90A-NEXT: s_cselect_b32 s4, s6, s4
-; GFX90A-NEXT: s_cselect_b32 s2, s5, s2
-; GFX90A-NEXT: s_add_i32 s5, s4, 1
-; GFX90A-NEXT: s_cmp_ge_u32 s2, s3
-; GFX90A-NEXT: s_cselect_b32 s2, s5, s4
-; GFX90A-NEXT: v_mov_b32_e32 v0, s2
-; GFX90A-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX90A-NEXT: v_readfirstlane_b32 s1, v0
+; GFX90A-NEXT: s_mul_i32 s0, s0, s1
+; GFX90A-NEXT: s_mul_hi_u32 s0, s1, s0
+; GFX90A-NEXT: s_add_i32 s1, s1, s0
+; GFX90A-NEXT: s_mul_hi_u32 s0, s6, s1
+; GFX90A-NEXT: s_mul_i32 s1, s0, s7
+; GFX90A-NEXT: s_sub_i32 s1, s6, s1
+; GFX90A-NEXT: s_add_i32 s2, s0, 1
+; GFX90A-NEXT: s_sub_i32 s3, s1, s7
+; GFX90A-NEXT: s_cmp_ge_u32 s1, s7
+; GFX90A-NEXT: s_cselect_b32 s0, s2, s0
+; GFX90A-NEXT: s_cselect_b32 s1, s3, s1
+; GFX90A-NEXT: s_add_i32 s2, s0, 1
+; GFX90A-NEXT: s_cmp_ge_u32 s1, s7
+; GFX90A-NEXT: s_cselect_b32 s0, s2, s0
+; GFX90A-NEXT: v_mov_b32_e32 v0, s0
+; GFX90A-NEXT: global_store_dword v1, v0, s[4:5]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: udiv_i32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s3
-; GFX10-NEXT: s_sub_i32 s5, 0, s3
+; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s7
+; GFX10-NEXT: s_sub_i32 s1, 0, s7
; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX10-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_mul_i32 s5, s5, s4
-; GFX10-NEXT: s_mul_hi_u32 s5, s4, s5
-; GFX10-NEXT: s_add_i32 s4, s4, s5
-; GFX10-NEXT: s_mul_hi_u32 s4, s2, s4
-; GFX10-NEXT: s_mul_i32 s5, s4, s3
-; GFX10-NEXT: s_sub_i32 s2, s2, s5
-; GFX10-NEXT: s_add_i32 s5, s4, 1
-; GFX10-NEXT: s_sub_i32 s6, s2, s3
-; GFX10-NEXT: s_cmp_ge_u32 s2, s3
-; GFX10-NEXT: s_cselect_b32 s4, s5, s4
-; GFX10-NEXT: s_cselect_b32 s2, s6, s2
-; GFX10-NEXT: s_add_i32 s5, s4, 1
-; GFX10-NEXT: s_cmp_ge_u32 s2, s3
-; GFX10-NEXT: s_cselect_b32 s2, s5, s4
-; GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_mul_i32 s1, s1, s0
+; GFX10-NEXT: s_mul_hi_u32 s1, s0, s1
+; GFX10-NEXT: s_add_i32 s0, s0, s1
+; GFX10-NEXT: s_mul_hi_u32 s0, s6, s0
+; GFX10-NEXT: s_mul_i32 s1, s0, s7
+; GFX10-NEXT: s_add_i32 s2, s0, 1
+; GFX10-NEXT: s_sub_i32 s1, s6, s1
+; GFX10-NEXT: s_sub_i32 s3, s1, s7
+; GFX10-NEXT: s_cmp_ge_u32 s1, s7
+; GFX10-NEXT: s_cselect_b32 s0, s2, s0
+; GFX10-NEXT: s_cselect_b32 s1, s3, s1
+; GFX10-NEXT: s_add_i32 s2, s0, 1
+; GFX10-NEXT: s_cmp_ge_u32 s1, s7
+; GFX10-NEXT: s_cselect_b32 s0, s2, s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s0
+; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: s_endpgm
;
; GFX9-FLATSCR-LABEL: udiv_i32:
; GFX9-FLATSCR: ; %bb.0:
-; GFX9-FLATSCR-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-FLATSCR-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-FLATSCR-NEXT: v_cvt_f32_u32_e32 v0, s3
-; GFX9-FLATSCR-NEXT: s_sub_i32 s4, 0, s3
+; GFX9-FLATSCR-NEXT: v_cvt_f32_u32_e32 v0, s7
+; GFX9-FLATSCR-NEXT: s_sub_i32 s0, 0, s7
; GFX9-FLATSCR-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX9-FLATSCR-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-FLATSCR-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-FLATSCR-NEXT: v_readfirstlane_b32 s5, v0
-; GFX9-FLATSCR-NEXT: s_mul_i32 s4, s4, s5
-; GFX9-FLATSCR-NEXT: s_mul_hi_u32 s4, s5, s4
-; GFX9-FLATSCR-NEXT: s_add_i32 s5, s5, s4
-; GFX9-FLATSCR-NEXT: s_mul_hi_u32 s4, s2, s5
-; GFX9-FLATSCR-NEXT: s_mul_i32 s5, s4, s3
-; GFX9-FLATSCR-NEXT: s_sub_i32 s2, s2, s5
-; GFX9-FLATSCR-NEXT: s_add_i32 s6, s4, 1
-; GFX9-FLATSCR-NEXT: s_sub_i32 s5, s2, s3
-; GFX9-FLATSCR-NEXT: s_cmp_ge_u32 s2, s3
-; GFX9-FLATSCR-NEXT: s_cselect_b32 s4, s6, s4
-; GFX9-FLATSCR-NEXT: s_cselect_b32 s2, s5, s2
-; GFX9-FLATSCR-NEXT: s_add_i32 s5, s4, 1
-; GFX9-FLATSCR-NEXT: s_cmp_ge_u32 s2, s3
-; GFX9-FLATSCR-NEXT: s_cselect_b32 s2, s5, s4
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-FLATSCR-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-FLATSCR-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-FLATSCR-NEXT: s_mul_i32 s0, s0, s1
+; GFX9-FLATSCR-NEXT: s_mul_hi_u32 s0, s1, s0
+; GFX9-FLATSCR-NEXT: s_add_i32 s1, s1, s0
+; GFX9-FLATSCR-NEXT: s_mul_hi_u32 s0, s6, s1
+; GFX9-FLATSCR-NEXT: s_mul_i32 s1, s0, s7
+; GFX9-FLATSCR-NEXT: s_sub_i32 s1, s6, s1
+; GFX9-FLATSCR-NEXT: s_add_i32 s2, s0, 1
+; GFX9-FLATSCR-NEXT: s_sub_i32 s3, s1, s7
+; GFX9-FLATSCR-NEXT: s_cmp_ge_u32 s1, s7
+; GFX9-FLATSCR-NEXT: s_cselect_b32 s0, s2, s0
+; GFX9-FLATSCR-NEXT: s_cselect_b32 s1, s3, s1
+; GFX9-FLATSCR-NEXT: s_add_i32 s2, s0, 1
+; GFX9-FLATSCR-NEXT: s_cmp_ge_u32 s1, s7
+; GFX9-FLATSCR-NEXT: s_cselect_b32 s0, s2, s0
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-FLATSCR-NEXT: global_store_dword v1, v0, s[4:5]
; GFX9-FLATSCR-NEXT: s_waitcnt vmcnt(0)
; GFX9-FLATSCR-NEXT: s_endpgm
;
@@ -1660,3 +1660,4 @@ entry:
%bc = bitcast <2 x i32> %r.1 to <2 x float>
ret <2 x float> %bc
}
+
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll
index e1caf3bea6119..77722d96d5a4c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll
@@ -32,21 +32,21 @@ define amdgpu_kernel void @s_cvt_pkrtz_v2f16_f32(ptr addrspace(1) %out, float %x
;
; GFX9-LABEL: s_cvt_pkrtz_v2f16_f32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: v_cvt_pkrtz_f16_f32 v1, s2, v1
-; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: v_cvt_pkrtz_f16_f32 v1, s6, v1
+; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: s_cvt_pkrtz_v2f16_f32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_cvt_pkrtz_f16_f32_e64 v1, s2, s3
-; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: v_cvt_pkrtz_f16_f32_e64 v1, s6, s7
+; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_cvt_pkrtz_v2f16_f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
index a26b84e17374a..d70df38fe6037 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
@@ -27,12 +27,12 @@ define amdgpu_kernel void @v_fcmp_f32_oeq_with_fabs(ptr addrspace(1) %out, float
;
; SDAG-GFX10-LABEL: v_fcmp_f32_oeq_with_fabs:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-GFX10-NEXT: v_cmp_eq_f32_e64 s2, s2, |s3|
-; SDAG-GFX10-NEXT: v_mov_b32_e32 v1, s2
-; SDAG-GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; SDAG-GFX10-NEXT: v_cmp_eq_f32_e64 s0, s6, |s7|
+; SDAG-GFX10-NEXT: v_mov_b32_e32 v1, s0
+; SDAG-GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; SDAG-GFX10-NEXT: s_endpgm
;
; GISEL-GFX11-LABEL: v_fcmp_f32_oeq_with_fabs:
@@ -50,12 +50,12 @@ define amdgpu_kernel void @v_fcmp_f32_oeq_with_fabs(ptr addrspace(1) %out, float
;
; GISEL-GFX10-LABEL: v_fcmp_f32_oeq_with_fabs:
; GISEL-GFX10: ; %bb.0:
-; GISEL-GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GISEL-GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-GFX10-NEXT: v_cmp_eq_f32_e64 s2, s2, |s3|
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s2
-; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GISEL-GFX10-NEXT: v_cmp_eq_f32_e64 s0, s6, |s7|
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GISEL-GFX10-NEXT: s_endpgm
%temp = call float @llvm.fabs.f32(float %a)
%result = call i32 @llvm.amdgcn.fcmp.f32(float %src, float %temp, i32 1)
@@ -78,12 +78,12 @@ define amdgpu_kernel void @v_fcmp_f32_oeq_both_operands_with_fabs(ptr addrspace(
;
; SDAG-GFX10-LABEL: v_fcmp_f32_oeq_both_operands_with_fabs:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-GFX10-NEXT: v_cmp_eq_f32_e64 s2, |s2|, |s3|
-; SDAG-GFX10-NEXT: v_mov_b32_e32 v1, s2
-; SDAG-GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; SDAG-GFX10-NEXT: v_cmp_eq_f32_e64 s0, |s6|, |s7|
+; SDAG-GFX10-NEXT: v_mov_b32_e32 v1, s0
+; SDAG-GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; SDAG-GFX10-NEXT: s_endpgm
;
; GISEL-GFX11-LABEL: v_fcmp_f32_oeq_both_operands_with_fabs:
@@ -101,12 +101,12 @@ define amdgpu_kernel void @v_fcmp_f32_oeq_both_operands_with_fabs(ptr addrspace(
;
; GISEL-GFX10-LABEL: v_fcmp_f32_oeq_both_operands_with_fabs:
; GISEL-GFX10: ; %bb.0:
-; GISEL-GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GISEL-GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GISEL-GFX10-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-GFX10-NEXT: v_cmp_eq_f32_e64 s2, |s2|, |s3|
-; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s2
-; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GISEL-GFX10-NEXT: v_cmp_eq_f32_e64 s0, |s6|, |s7|
+; GISEL-GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GISEL-GFX10-NEXT: s_endpgm
%temp = call float @llvm.fabs.f32(float %a)
%src_input = call float @llvm.fabs.f32(float %src)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
index 7e78d8b05d09f..734d1472d054f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
@@ -30,14 +30,14 @@ define amdgpu_kernel void @v_fcmp_f32_oeq_with_fabs(ptr addrspace(1) %out, float
;
; GFX9-LABEL: v_fcmp_f32_oeq_with_fabs:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_cmp_eq_f32_e64 s[2:3], s2, |v0|
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v0, s7
+; GFX9-NEXT: v_cmp_eq_f32_e64 s[0:1], s6, |v0|
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX9-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f32_oeq_with_fabs:
@@ -88,14 +88,14 @@ define amdgpu_kernel void @v_fcmp_f32_oeq_both_operands_with_fabs(ptr addrspace(
;
; GFX9-LABEL: v_fcmp_f32_oeq_both_operands_with_fabs:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_cmp_eq_f32_e64 s[2:3], |s2|, |v0|
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v0, s7
+; GFX9-NEXT: v_cmp_eq_f32_e64 s[0:1], |s6|, |v0|
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX9-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f32_oeq_both_operands_with_fabs:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.atomic.csub.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.atomic.csub.ll
index d7dd0ce58a08f..4a66b761306f3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.atomic.csub.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.atomic.csub.ll
@@ -6,7 +6,7 @@ declare i32 @llvm.amdgcn.global.atomic.csub(ptr addrspace(1), i32)
; GCN-LABEL: {{^}}global_atomic_csub_rtn:
; PREGFX12: global_atomic_csub v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9:]+}}, s{{\[[0-9]+:[0-9]+\]}} glc
-; GFX12PLUS: global_atomic_sub_clamp_u32 v0, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX12PLUS: global_atomic_sub_clamp_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}} th:TH_ATOMIC_RETURN
define amdgpu_kernel void @global_atomic_csub_rtn(ptr addrspace(1) %ptr, i32 %data) {
main_body:
%ret = call i32 @llvm.amdgcn.global.atomic.csub(ptr addrspace(1) %ptr, i32 %data)
@@ -15,7 +15,7 @@ main_body:
; GCN-LABEL: {{^}}global_atomic_csub_no_rtn:
; PREGFX12: global_atomic_csub v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}
-; GFX12PLUS: global_atomic_sub_clamp_u32 v0, v1, s[0:1]
+; GFX12PLUS: global_atomic_sub_clamp_u32 v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}
define amdgpu_kernel void @global_atomic_csub_no_rtn(ptr addrspace(1) %ptr, i32 %data) #0 {
main_body:
%ret = call i32 @llvm.amdgcn.global.atomic.csub(ptr addrspace(1) %ptr, i32 %data)
@@ -24,7 +24,7 @@ main_body:
; GCN-LABEL: {{^}}global_atomic_csub_off4_rtn:
; PREGFX12: global_atomic_csub v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}} offset:4 glc
-; GFX12PLUS: global_atomic_sub_clamp_u32 v0, v0, v1, s[0:1] offset:4 th:TH_ATOMIC_RETURN
+; GFX12PLUS: global_atomic_sub_clamp_u32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}} offset:4 th:TH_ATOMIC_RETURN
define amdgpu_kernel void @global_atomic_csub_off4_rtn(ptr addrspace(1) %ptr, i32 %data) {
main_body:
%p = getelementptr i32, ptr addrspace(1) %ptr, i64 1
@@ -34,7 +34,7 @@ main_body:
; GCN-LABEL: {{^}}global_atomic_csub_off4_no_rtn:
; PREGFX12: global_atomic_csub v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}} offset:4
-; GFX12PLUS: global_atomic_sub_clamp_u32 v0, v1, s[0:1] offset:4
+; GFX12PLUS: global_atomic_sub_clamp_u32 v{{[0-9]+}}, v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}} offset:4
define amdgpu_kernel void @global_atomic_csub_off4_no_rtn(ptr addrspace(1) %ptr, i32 %data) #0 {
main_body:
%p = getelementptr i32, ptr addrspace(1) %ptr, i64 1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
index 9e3e393d82e22..746b00ba7b231 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
@@ -1759,16 +1759,16 @@ define amdgpu_kernel void @v_icmp_i1_ne0(ptr addrspace(1) %out, i32 %a, i32 %b)
;
; GFX10-LABEL: v_icmp_i1_ne0:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_cmp_gt_u32 s2, 1
-; GFX10-NEXT: s_cselect_b32 s2, -1, 0
-; GFX10-NEXT: s_cmp_gt_u32 s3, 2
-; GFX10-NEXT: s_cselect_b32 s3, -1, 0
-; GFX10-NEXT: s_and_b32 s2, s2, s3
-; GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_cmp_gt_u32 s6, 1
+; GFX10-NEXT: s_cselect_b32 s0, -1, 0
+; GFX10-NEXT: s_cmp_gt_u32 s7, 2
+; GFX10-NEXT: s_cselect_b32 s1, -1, 0
+; GFX10-NEXT: s_and_b32 s0, s0, s1
+; GFX10-NEXT: v_mov_b32_e32 v1, s0
+; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
%c0 = icmp ugt i32 %a, 1
%c1 = icmp ugt i32 %b, 2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
index 60e242bf5b0e8..2625c1f152219 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
@@ -1986,17 +1986,17 @@ define amdgpu_kernel void @v_icmp_i1_ne0(ptr addrspace(1) %out, i32 %a, i32 %b)
;
; GFX9-LABEL: v_icmp_i1_ne0:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_cmp_gt_u32 s2, 1
-; GFX9-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX9-NEXT: s_cmp_gt_u32 s3, 2
+; GFX9-NEXT: s_cmp_gt_u32 s6, 1
+; GFX9-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX9-NEXT: s_cmp_gt_u32 s7, 2
; GFX9-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX9-NEXT: s_and_b64 s[2:3], s[4:5], s[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX9-NEXT: s_endpgm
%c0 = icmp ugt i32 %a, 1
%c1 = icmp ugt i32 %b, 2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
index 2d01703c78d78..4930317143a76 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
@@ -163,7 +163,7 @@ main_body:
define amdgpu_kernel void @image_bvh_intersect_ray_nsa_reassign(ptr %p_node_ptr, ptr %p_ray, <4 x i32> inreg %tdescr) {
; GFX1013-LABEL: image_bvh_intersect_ray_nsa_reassign:
; GFX1013: ; %bb.0: ; %main_body
-; GFX1013-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
+; GFX1013-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; GFX1013-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1013-NEXT: v_mov_b32_e32 v6, 4.0
; GFX1013-NEXT: v_mov_b32_e32 v7, 0x40a00000
@@ -171,10 +171,10 @@ define amdgpu_kernel void @image_bvh_intersect_ray_nsa_reassign(ptr %p_node_ptr,
; GFX1013-NEXT: v_mov_b32_e32 v9, 0x40e00000
; GFX1013-NEXT: v_mov_b32_e32 v10, 0x41000000
; GFX1013-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1013-NEXT: v_add_co_u32 v2, s0, s0, v0
-; GFX1013-NEXT: v_add_co_ci_u32_e64 v3, s0, s1, 0, s0
-; GFX1013-NEXT: v_add_co_u32 v4, s0, s2, v0
-; GFX1013-NEXT: v_add_co_ci_u32_e64 v5, s0, s3, 0, s0
+; GFX1013-NEXT: v_add_co_u32 v2, s0, s4, v0
+; GFX1013-NEXT: v_add_co_ci_u32_e64 v3, s0, s5, 0, s0
+; GFX1013-NEXT: v_add_co_u32 v4, s0, s6, v0
+; GFX1013-NEXT: v_add_co_ci_u32_e64 v5, s0, s7, 0, s0
; GFX1013-NEXT: flat_load_dword v0, v[2:3]
; GFX1013-NEXT: flat_load_dword v1, v[4:5]
; GFX1013-NEXT: v_mov_b32_e32 v2, 0
@@ -182,7 +182,7 @@ define amdgpu_kernel void @image_bvh_intersect_ray_nsa_reassign(ptr %p_node_ptr,
; GFX1013-NEXT: v_mov_b32_e32 v4, 2.0
; GFX1013-NEXT: v_mov_b32_e32 v5, 0x40400000
; GFX1013-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX1013-NEXT: image_bvh_intersect_ray v[0:3], v[0:10], s[4:7]
+; GFX1013-NEXT: image_bvh_intersect_ray v[0:3], v[0:10], s[8:11]
; GFX1013-NEXT: s_waitcnt vmcnt(0)
; GFX1013-NEXT: flat_store_dwordx4 v[0:1], v[0:3]
; GFX1013-NEXT: s_endpgm
@@ -262,15 +262,15 @@ main_body:
define amdgpu_kernel void @image_bvh_intersect_ray_a16_nsa_reassign(ptr %p_node_ptr, ptr %p_ray, <4 x i32> inreg %tdescr) {
; GFX1013-LABEL: image_bvh_intersect_ray_a16_nsa_reassign:
; GFX1013: ; %bb.0: ; %main_body
-; GFX1013-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
+; GFX1013-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; GFX1013-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1013-NEXT: v_mov_b32_e32 v6, 0x46004500
; GFX1013-NEXT: v_mov_b32_e32 v7, 0x48004700
; GFX1013-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1013-NEXT: v_add_co_u32 v2, s0, s0, v0
-; GFX1013-NEXT: v_add_co_ci_u32_e64 v3, s0, s1, 0, s0
-; GFX1013-NEXT: v_add_co_u32 v4, s0, s2, v0
-; GFX1013-NEXT: v_add_co_ci_u32_e64 v5, s0, s3, 0, s0
+; GFX1013-NEXT: v_add_co_u32 v2, s0, s4, v0
+; GFX1013-NEXT: v_add_co_ci_u32_e64 v3, s0, s5, 0, s0
+; GFX1013-NEXT: v_add_co_u32 v4, s0, s6, v0
+; GFX1013-NEXT: v_add_co_ci_u32_e64 v5, s0, s7, 0, s0
; GFX1013-NEXT: flat_load_dword v0, v[2:3]
; GFX1013-NEXT: flat_load_dword v1, v[4:5]
; GFX1013-NEXT: v_mov_b32_e32 v2, 0
@@ -278,7 +278,7 @@ define amdgpu_kernel void @image_bvh_intersect_ray_a16_nsa_reassign(ptr %p_node_
; GFX1013-NEXT: v_mov_b32_e32 v4, 2.0
; GFX1013-NEXT: v_mov_b32_e32 v5, 0x44004200
; GFX1013-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX1013-NEXT: image_bvh_intersect_ray v[0:3], v[0:7], s[4:7] a16
+; GFX1013-NEXT: image_bvh_intersect_ray v[0:3], v[0:7], s[8:11] a16
; GFX1013-NEXT: s_waitcnt vmcnt(0)
; GFX1013-NEXT: flat_store_dwordx4 v[0:1], v[0:3]
; GFX1013-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
index 8e9a652ae8a8e..aad74410d1453 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.ll
@@ -384,26 +384,26 @@ define amdgpu_kernel void @v_permlane16_b32_vii_f32(ptr addrspace(1) %out, float
define amdgpu_kernel void @v_permlane16_b32_vii_i64(ptr addrspace(1) %out, i64 %src0) {
; GFX10-SDAG-LABEL: v_permlane16_b32_vii_i64:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s6
; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, 1, 2
; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, 1, 2
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlane16_b32_vii_i64:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s7
; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, 1, 2
; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, 1, 2
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vii_i64:
@@ -469,26 +469,26 @@ define amdgpu_kernel void @v_permlane16_b32_vii_i64(ptr addrspace(1) %out, i64 %
define amdgpu_kernel void @v_permlane16_b32_vii_f64(ptr addrspace(1) %out, double %src0) {
; GFX10-SDAG-LABEL: v_permlane16_b32_vii_f64:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s6
; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, 1, 2
; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, 1, 2
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlane16_b32_vii_f64:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s7
; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, 1, 2
; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, 1, 2
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vii_f64:
@@ -601,28 +601,28 @@ define amdgpu_kernel void @v_permlane16_b32_vll_i32(ptr addrspace(1) %out, i32 %
define amdgpu_kernel void @v_permlane16_b32_vll_i64(ptr addrspace(1) %out, i64 %src0) {
; GFX10-SDAG-LABEL: v_permlane16_b32_vll_i64:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-SDAG-NEXT: s_movk_i32 s0, 0x1234
; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: s_movk_i32 s2, 0x1234
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s2, 0xc1d1
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s2, 0xc1d1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, 0xc1d1
+; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, 0xc1d1
+; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlane16_b32_vll_i64:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_movk_i32 s0, 0x1234
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: s_movk_i32 s2, 0x1234
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s2, 0xc1d1
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s2, 0xc1d1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, 0xc1d1
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, 0xc1d1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vll_i64:
@@ -742,28 +742,28 @@ define amdgpu_kernel void @v_permlane16_b32_vll_f32(ptr addrspace(1) %out,float
define amdgpu_kernel void @v_permlane16_b32_vll_f64(ptr addrspace(1) %out, double %src0) {
; GFX10-SDAG-LABEL: v_permlane16_b32_vll_f64:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-SDAG-NEXT: s_movk_i32 s0, 0x1234
; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: s_movk_i32 s2, 0x1234
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s2, 0xc1d1
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s2, 0xc1d1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, 0xc1d1
+; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, 0xc1d1
+; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlane16_b32_vll_f64:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_movk_i32 s0, 0x1234
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: s_movk_i32 s2, 0x1234
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s2, 0xc1d1
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s2, 0xc1d1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, 0xc1d1
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, 0xc1d1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vll_f64:
@@ -933,30 +933,30 @@ define amdgpu_kernel void @v_permlane16_b32_vvv_i32(ptr addrspace(1) %out, i32 %
define amdgpu_kernel void @v_permlane16_b32_vvv_i64(ptr addrspace(1) %out, i64 %src0) {
; GFX10-SDAG-LABEL: v_permlane16_b32_vvv_i64:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-SDAG-NEXT: v_readfirstlane_b32 s1, v1
; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s4, s5
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1
+; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlane16_b32_vvv_i64:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s1, v1
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s4, s5
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-LABEL: v_permlane16_b32_vvv_i64:
@@ -1104,30 +1104,30 @@ define amdgpu_kernel void @v_permlane16_b32_vvv_f32(ptr addrspace(1) %out, float
define amdgpu_kernel void @v_permlane16_b32_vvv_f64(ptr addrspace(1) %out, double %src0) {
; GFX10-SDAG-LABEL: v_permlane16_b32_vvv_f64:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-SDAG-NEXT: v_readfirstlane_b32 s1, v1
; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s4, s5
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s4, s5
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s1
+; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlane16_b32_vvv_f64:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s1, v1
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s4, s5
-; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s4, s5
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s1
+; GFX10-GISEL-NEXT: v_permlane16_b32 v1, v1, s0, s1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-LABEL: v_permlane16_b32_vvv_f64:
@@ -1179,24 +1179,24 @@ define amdgpu_kernel void @v_permlane16_b32_vvv_f64(ptr addrspace(1) %out, doubl
define amdgpu_kernel void @v_permlane16_b32_vvs_i32(ptr addrspace(1) %out, i32 %src0, i32 %src2) {
; GFX10-SDAG-LABEL: v_permlane16_b32_vvs_i32:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s2, v0
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-SDAG-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s2, s3
-; GFX10-SDAG-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s6
+; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s7
+; GFX10-SDAG-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlane16_b32_vvs_i32:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s4, s3
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s7
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vvs_i32:
@@ -1337,24 +1337,24 @@ define amdgpu_kernel void @v_permlane16_b32_vvs_i64(ptr addrspace(1) %out, i64 %
define amdgpu_kernel void @v_permlane16_b32_vvs_f32(ptr addrspace(1) %out, float %src0, i32 %src2) {
; GFX10-SDAG-LABEL: v_permlane16_b32_vvs_f32:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s2, v0
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-SDAG-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s2, s3
-; GFX10-SDAG-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s6
+; GFX10-SDAG-NEXT: v_permlane16_b32 v1, v1, s0, s7
+; GFX10-SDAG-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlane16_b32_vvs_f32:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s4, s3
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s0, s7
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vvs_f32:
@@ -1493,27 +1493,16 @@ define amdgpu_kernel void @v_permlane16_b32_vvs_f64(ptr addrspace(1) %out, doubl
}
define amdgpu_kernel void @v_permlane16_b32_vsv_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vsv_i32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s2, v1
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s3, s2
-; GFX10-SDAG-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vsv_i32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s3, s4
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vsv_i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-NEXT: v_readfirstlane_b32 s0, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s7, s0
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vsv_i32:
; GFX11-SDAG: ; %bb.0:
@@ -1693,27 +1682,16 @@ define amdgpu_kernel void @v_permlane16_b32_vsv_i64(ptr addrspace(1) %out, i64 %
}
define amdgpu_kernel void @v_permlane16_b32_vsv_f32(ptr addrspace(1) %out, float %src0, i32 %src1) {
-; GFX10-SDAG-LABEL: v_permlane16_b32_vsv_f32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s2, v1
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-SDAG-NEXT: v_permlane16_b32 v0, v0, s3, s2
-; GFX10-SDAG-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlane16_b32_vsv_f32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_permlane16_b32 v0, v0, s3, s4
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlane16_b32_vsv_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-NEXT: v_readfirstlane_b32 s0, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-NEXT: v_permlane16_b32 v0, v0, s7, s0
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlane16_b32_vsv_f32:
; GFX11-SDAG: ; %bb.0:
@@ -3117,26 +3095,26 @@ define amdgpu_kernel void @v_permlanex16_b32_vii_f32(ptr addrspace(1) %out, floa
define amdgpu_kernel void @v_permlanex16_b32_vii_i64(ptr addrspace(1) %out, i64 %src0) {
; GFX10-SDAG-LABEL: v_permlanex16_b32_vii_i64:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s6
; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, 1, 2
; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, 1, 2
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlanex16_b32_vii_i64:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s7
; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, 1, 2
; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, 1, 2
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vii_i64:
@@ -3202,26 +3180,26 @@ define amdgpu_kernel void @v_permlanex16_b32_vii_i64(ptr addrspace(1) %out, i64
define amdgpu_kernel void @v_permlanex16_b32_vii_f64(ptr addrspace(1) %out, double %src0) {
; GFX10-SDAG-LABEL: v_permlanex16_b32_vii_f64:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s6
; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, 1, 2
; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, 1, 2
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlanex16_b32_vii_f64:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s7
; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, 1, 2
; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, 1, 2
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vii_f64:
@@ -3380,28 +3358,28 @@ define amdgpu_kernel void @v_permlanex16_b32_vll_f32(ptr addrspace(1) %out, floa
define amdgpu_kernel void @v_permlanex16_b32_vll_i64(ptr addrspace(1) %out, i64 %src0) {
; GFX10-SDAG-LABEL: v_permlanex16_b32_vll_i64:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-SDAG-NEXT: s_movk_i32 s0, 0x1234
; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: s_movk_i32 s2, 0x1234
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s2, 0xc1d1
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s2, 0xc1d1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, 0xc1d1
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, 0xc1d1
+; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlanex16_b32_vll_i64:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_movk_i32 s0, 0x1234
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: s_movk_i32 s2, 0x1234
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s2, 0xc1d1
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s2, 0xc1d1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, 0xc1d1
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, 0xc1d1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vll_i64:
@@ -3475,28 +3453,28 @@ define amdgpu_kernel void @v_permlanex16_b32_vll_i64(ptr addrspace(1) %out, i64
define amdgpu_kernel void @v_permlanex16_b32_vll_f64(ptr addrspace(1) %out, double %src0) {
; GFX10-SDAG-LABEL: v_permlanex16_b32_vll_f64:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-SDAG-NEXT: s_movk_i32 s0, 0x1234
; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: s_movk_i32 s2, 0x1234
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s2, 0xc1d1
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s2, 0xc1d1
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, 0xc1d1
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, 0xc1d1
+; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlanex16_b32_vll_f64:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-GISEL-NEXT: s_movk_i32 s0, 0x1234
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: s_movk_i32 s2, 0x1234
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s2, 0xc1d1
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s2, 0xc1d1
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, 0xc1d1
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, 0xc1d1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vll_f64:
@@ -3762,30 +3740,30 @@ define amdgpu_kernel void @v_permlanex16_b32_vvv_f32(ptr addrspace(1) %out, floa
define amdgpu_kernel void @v_permlanex16_b32_vvv_i64(ptr addrspace(1) %out, i64 %src0) {
; GFX10-SDAG-LABEL: v_permlanex16_b32_vvv_i64:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-SDAG-NEXT: v_readfirstlane_b32 s1, v1
; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s4, s5
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlanex16_b32_vvv_i64:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s1, v1
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s4, s5
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-LABEL: v_permlanex16_b32_vvv_i64:
@@ -3837,30 +3815,30 @@ define amdgpu_kernel void @v_permlanex16_b32_vvv_i64(ptr addrspace(1) %out, i64
define amdgpu_kernel void @v_permlanex16_b32_vvv_f64(ptr addrspace(1) %out, double %src0) {
; GFX10-SDAG-LABEL: v_permlanex16_b32_vvv_f64:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-SDAG-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-SDAG-NEXT: v_readfirstlane_b32 s1, v1
; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s4, s5
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s4, s5
-; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s1
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX10-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlanex16_b32_vvv_f64:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s5, v1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s1, v1
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s4, s5
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s4, s5
-; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s1
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v1, v1, s0, s1
+; GFX10-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-LABEL: v_permlanex16_b32_vvv_f64:
@@ -3912,24 +3890,24 @@ define amdgpu_kernel void @v_permlanex16_b32_vvv_f64(ptr addrspace(1) %out, doub
define amdgpu_kernel void @v_permlanex16_b32_vvs_i32(ptr addrspace(1) %out, i32 %src0, i32 %src2) {
; GFX10-SDAG-LABEL: v_permlanex16_b32_vvs_i32:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s2, v0
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-SDAG-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s2, s3
-; GFX10-SDAG-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s6
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s7
+; GFX10-SDAG-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlanex16_b32_vvs_i32:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s4, s3
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s7
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vvs_i32:
@@ -3996,24 +3974,24 @@ define amdgpu_kernel void @v_permlanex16_b32_vvs_i32(ptr addrspace(1) %out, i32
define amdgpu_kernel void @v_permlanex16_b32_vvs_f32(ptr addrspace(1) %out, float %src0, i32 %src2) {
; GFX10-SDAG-LABEL: v_permlanex16_b32_vvs_f32:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s2, v0
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-SDAG-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s2, s3
-; GFX10-SDAG-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s6
+; GFX10-SDAG-NEXT: v_permlanex16_b32 v1, v1, s0, s7
+; GFX10-SDAG-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: v_permlanex16_b32_vvs_f32:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-GISEL-NEXT: v_readfirstlane_b32 s0, v0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s4, s3
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s0, s7
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vvs_f32:
@@ -4226,27 +4204,16 @@ define amdgpu_kernel void @v_permlanex16_b32_vvs_f64(ptr addrspace(1) %out, doub
}
define amdgpu_kernel void @v_permlanex16_b32_vsv_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vsv_i32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s2, v1
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s3, s2
-; GFX10-SDAG-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vsv_i32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s3, s4
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vsv_i32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-NEXT: v_readfirstlane_b32 s0, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s7, s0
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vsv_i32:
; GFX11-SDAG: ; %bb.0:
@@ -4314,27 +4281,16 @@ define amdgpu_kernel void @v_permlanex16_b32_vsv_i32(ptr addrspace(1) %out, i32
}
define amdgpu_kernel void @v_permlanex16_b32_vsv_f32(ptr addrspace(1) %out, float %src0, i32 %src1) {
-; GFX10-SDAG-LABEL: v_permlanex16_b32_vsv_f32:
-; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_readfirstlane_b32 s2, v1
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-SDAG-NEXT: v_permlanex16_b32 v0, v0, s3, s2
-; GFX10-SDAG-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-SDAG-NEXT: s_endpgm
-;
-; GFX10-GISEL-LABEL: v_permlanex16_b32_vsv_f32:
-; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-GISEL-NEXT: v_readfirstlane_b32 s4, v1
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-GISEL-NEXT: v_permlanex16_b32 v0, v0, s3, s4
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[0:1]
-; GFX10-GISEL-NEXT: s_endpgm
+; GFX10-LABEL: v_permlanex16_b32_vsv_f32:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-NEXT: v_readfirstlane_b32 s0, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-NEXT: v_permlanex16_b32 v0, v0, s7, s0
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
+; GFX10-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: v_permlanex16_b32_vsv_f32:
; GFX11-SDAG: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
index 24b8a3c2dc873..b8a4674833cee 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.sched.group.barrier.ll
@@ -29,19 +29,19 @@ entry:
define amdgpu_kernel void @test_sched_group_barrier_pipeline_READ_VALU_WRITE(ptr addrspace(1) noalias %in, ptr addrspace(1) noalias %out) #0 {
; GCN-LABEL: test_sched_group_barrier_pipeline_READ_VALU_WRITE:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GCN-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GCN-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GCN-NEXT: v_lshlrev_b32_e32 v32, 7, v0
-; GCN-NEXT: ; kill: killed $sgpr0_sgpr1
+; GCN-NEXT: ; kill: killed $sgpr4_sgpr5
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
-; GCN-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
-; GCN-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
-; GCN-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
-; GCN-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64
-; GCN-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
-; GCN-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96
-; GCN-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112
+; GCN-NEXT: global_load_dwordx4 v[0:3], v32, s[4:5]
+; GCN-NEXT: global_load_dwordx4 v[4:7], v32, s[4:5] offset:16
+; GCN-NEXT: global_load_dwordx4 v[8:11], v32, s[4:5] offset:32
+; GCN-NEXT: global_load_dwordx4 v[12:15], v32, s[4:5] offset:48
+; GCN-NEXT: global_load_dwordx4 v[16:19], v32, s[4:5] offset:64
+; GCN-NEXT: global_load_dwordx4 v[20:23], v32, s[4:5] offset:80
+; GCN-NEXT: global_load_dwordx4 v[24:27], v32, s[4:5] offset:96
+; GCN-NEXT: global_load_dwordx4 v[28:31], v32, s[4:5] offset:112
; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(8) SyncID(0)
; GCN-NEXT: s_waitcnt vmcnt(7)
; GCN-NEXT: v_mul_lo_u32 v3, v3, v3
@@ -83,33 +83,33 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_READ_VALU_WRITE(ptr
; GCN-NEXT: v_mul_lo_u32 v30, v30, v30
; GCN-NEXT: v_mul_lo_u32 v29, v29, v29
; GCN-NEXT: v_mul_lo_u32 v28, v28, v28
-; GCN-NEXT: global_store_dwordx4 v32, v[28:31], s[2:3] offset:112
-; GCN-NEXT: global_store_dwordx4 v32, v[24:27], s[2:3] offset:96
-; GCN-NEXT: global_store_dwordx4 v32, v[20:23], s[2:3] offset:80
-; GCN-NEXT: global_store_dwordx4 v32, v[16:19], s[2:3] offset:64
-; GCN-NEXT: global_store_dwordx4 v32, v[12:15], s[2:3] offset:48
-; GCN-NEXT: global_store_dwordx4 v32, v[8:11], s[2:3] offset:32
-; GCN-NEXT: global_store_dwordx4 v32, v[4:7], s[2:3] offset:16
-; GCN-NEXT: global_store_dwordx4 v32, v[0:3], s[2:3]
+; GCN-NEXT: global_store_dwordx4 v32, v[28:31], s[6:7] offset:112
+; GCN-NEXT: global_store_dwordx4 v32, v[24:27], s[6:7] offset:96
+; GCN-NEXT: global_store_dwordx4 v32, v[20:23], s[6:7] offset:80
+; GCN-NEXT: global_store_dwordx4 v32, v[16:19], s[6:7] offset:64
+; GCN-NEXT: global_store_dwordx4 v32, v[12:15], s[6:7] offset:48
+; GCN-NEXT: global_store_dwordx4 v32, v[8:11], s[6:7] offset:32
+; GCN-NEXT: global_store_dwordx4 v32, v[4:7], s[6:7] offset:16
+; GCN-NEXT: global_store_dwordx4 v32, v[0:3], s[6:7]
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(30) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(8) SyncID(0)
; GCN-NEXT: s_endpgm
;
; EXACTCUTOFF-LABEL: test_sched_group_barrier_pipeline_READ_VALU_WRITE:
; EXACTCUTOFF: ; %bb.0:
-; EXACTCUTOFF-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; EXACTCUTOFF-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; EXACTCUTOFF-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; EXACTCUTOFF-NEXT: v_lshlrev_b32_e32 v32, 7, v0
-; EXACTCUTOFF-NEXT: ; kill: killed $sgpr0_sgpr1
+; EXACTCUTOFF-NEXT: ; kill: killed $sgpr4_sgpr5
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:16
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:32
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:48
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:64
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:80
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:96
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:112
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[0:3], v32, s[4:5]
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[4:7], v32, s[4:5] offset:16
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[8:11], v32, s[4:5] offset:32
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[12:15], v32, s[4:5] offset:48
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[16:19], v32, s[4:5] offset:64
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[20:23], v32, s[4:5] offset:80
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[24:27], v32, s[4:5] offset:96
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[28:31], v32, s[4:5] offset:112
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(8) SyncID(0)
; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(7)
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v3, v3, v3
@@ -151,14 +151,14 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_READ_VALU_WRITE(ptr
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v30, v30, v30
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v29, v29, v29
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v28, v28, v28
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[28:31], s[2:3] offset:112
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[24:27], s[2:3] offset:96
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[20:23], s[2:3] offset:80
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[16:19], s[2:3] offset:64
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[12:15], s[2:3] offset:48
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[8:11], s[2:3] offset:32
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[4:7], s[2:3] offset:16
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[0:3], s[2:3]
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[28:31], s[6:7] offset:112
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[24:27], s[6:7] offset:96
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[20:23], s[6:7] offset:80
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[16:19], s[6:7] offset:64
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[12:15], s[6:7] offset:48
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[8:11], s[6:7] offset:32
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[4:7], s[6:7] offset:16
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[0:3], s[6:7]
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(30) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000040) size(8) SyncID(0)
; EXACTCUTOFF-NEXT: s_endpgm
@@ -180,12 +180,12 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_READ_VALU_WRITE(ptr
define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VALU(ptr addrspace(1) noalias %in, ptr addrspace(1) noalias %out) #0 {
; GCN-LABEL: test_sched_group_barrier_pipeline_alternating_READ_VALU:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GCN-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GCN-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GCN-NEXT: v_lshlrev_b32_e32 v32, 7, v0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:16
-; GCN-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
+; GCN-NEXT: global_load_dwordx4 v[28:31], v32, s[4:5] offset:16
+; GCN-NEXT: global_load_dwordx4 v[0:3], v32, s[4:5]
; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
@@ -194,10 +194,10 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_mul_lo_u32 v3, v3, v3
; GCN-NEXT: v_mul_lo_u32 v2, v2, v2
-; GCN-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:112
+; GCN-NEXT: global_load_dwordx4 v[4:7], v32, s[4:5] offset:112
; GCN-NEXT: v_mul_lo_u32 v1, v1, v1
; GCN-NEXT: v_mul_lo_u32 v0, v0, v0
-; GCN-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:96
+; GCN-NEXT: global_load_dwordx4 v[8:11], v32, s[4:5] offset:96
; GCN-NEXT: v_mul_lo_u32 v28, v28, v28
; GCN-NEXT: v_mul_lo_u32 v31, v31, v31
; GCN-NEXT: v_mul_lo_u32 v30, v30, v30
@@ -208,10 +208,10 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; GCN-NEXT: s_waitcnt vmcnt(1)
; GCN-NEXT: v_mul_lo_u32 v7, v7, v7
; GCN-NEXT: v_mul_lo_u32 v6, v6, v6
-; GCN-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:80
+; GCN-NEXT: global_load_dwordx4 v[12:15], v32, s[4:5] offset:80
; GCN-NEXT: v_mul_lo_u32 v5, v5, v5
; GCN-NEXT: v_mul_lo_u32 v4, v4, v4
-; GCN-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:48
+; GCN-NEXT: global_load_dwordx4 v[16:19], v32, s[4:5] offset:48
; GCN-NEXT: s_waitcnt vmcnt(2)
; GCN-NEXT: v_mul_lo_u32 v11, v11, v11
; GCN-NEXT: v_mul_lo_u32 v10, v10, v10
@@ -228,8 +228,8 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_mul_lo_u32 v19, v19, v19
; GCN-NEXT: v_mul_lo_u32 v18, v18, v18
-; GCN-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:64
-; GCN-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:32
+; GCN-NEXT: global_load_dwordx4 v[20:23], v32, s[4:5] offset:64
+; GCN-NEXT: global_load_dwordx4 v[24:27], v32, s[4:5] offset:32
; GCN-NEXT: v_mul_lo_u32 v17, v17, v17
; GCN-NEXT: v_mul_lo_u32 v16, v16, v16
; GCN-NEXT: v_mul_lo_u32 v12, v12, v12
@@ -245,14 +245,14 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; GCN-NEXT: v_mul_lo_u32 v22, v22, v22
; GCN-NEXT: v_mul_lo_u32 v21, v21, v21
; GCN-NEXT: v_mul_lo_u32 v20, v20, v20
-; GCN-NEXT: global_store_dwordx4 v32, v[4:7], s[2:3] offset:112
-; GCN-NEXT: global_store_dwordx4 v32, v[8:11], s[2:3] offset:96
-; GCN-NEXT: global_store_dwordx4 v32, v[12:15], s[2:3] offset:80
-; GCN-NEXT: global_store_dwordx4 v32, v[20:23], s[2:3] offset:64
-; GCN-NEXT: global_store_dwordx4 v32, v[16:19], s[2:3] offset:48
-; GCN-NEXT: global_store_dwordx4 v32, v[24:27], s[2:3] offset:32
-; GCN-NEXT: global_store_dwordx4 v32, v[28:31], s[2:3] offset:16
-; GCN-NEXT: global_store_dwordx4 v32, v[0:3], s[2:3]
+; GCN-NEXT: global_store_dwordx4 v32, v[4:7], s[6:7] offset:112
+; GCN-NEXT: global_store_dwordx4 v32, v[8:11], s[6:7] offset:96
+; GCN-NEXT: global_store_dwordx4 v32, v[12:15], s[6:7] offset:80
+; GCN-NEXT: global_store_dwordx4 v32, v[20:23], s[6:7] offset:64
+; GCN-NEXT: global_store_dwordx4 v32, v[16:19], s[6:7] offset:48
+; GCN-NEXT: global_store_dwordx4 v32, v[24:27], s[6:7] offset:32
+; GCN-NEXT: global_store_dwordx4 v32, v[28:31], s[6:7] offset:16
+; GCN-NEXT: global_store_dwordx4 v32, v[0:3], s[6:7]
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
@@ -261,12 +261,12 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
;
; EXACTCUTOFF-LABEL: test_sched_group_barrier_pipeline_alternating_READ_VALU:
; EXACTCUTOFF: ; %bb.0:
-; EXACTCUTOFF-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; EXACTCUTOFF-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; EXACTCUTOFF-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; EXACTCUTOFF-NEXT: v_lshlrev_b32_e32 v32, 7, v0
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[28:31], v32, s[0:1] offset:16
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[0:3], v32, s[0:1]
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[28:31], v32, s[4:5] offset:16
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[0:3], v32, s[4:5]
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
@@ -275,10 +275,10 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v3, v3, v3
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v2, v2, v2
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[4:7], v32, s[0:1] offset:112
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[4:7], v32, s[4:5] offset:112
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v1, v1, v1
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v0, v0, v0
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[8:11], v32, s[0:1] offset:96
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[8:11], v32, s[4:5] offset:96
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v28, v28, v28
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v31, v31, v31
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v30, v30, v30
@@ -289,10 +289,10 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(1)
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v7, v7, v7
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v6, v6, v6
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[12:15], v32, s[0:1] offset:80
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[12:15], v32, s[4:5] offset:80
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v5, v5, v5
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v4, v4, v4
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[16:19], v32, s[0:1] offset:48
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[16:19], v32, s[4:5] offset:48
; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(2)
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v11, v11, v11
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v10, v10, v10
@@ -309,8 +309,8 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v19, v19, v19
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v18, v18, v18
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[20:23], v32, s[0:1] offset:64
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[24:27], v32, s[0:1] offset:32
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[20:23], v32, s[4:5] offset:64
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[24:27], v32, s[4:5] offset:32
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v17, v17, v17
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v16, v16, v16
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v12, v12, v12
@@ -326,14 +326,14 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v22, v22, v22
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v21, v21, v21
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v20, v20, v20
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[4:7], s[2:3] offset:112
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[8:11], s[2:3] offset:96
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[12:15], s[2:3] offset:80
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[20:23], s[2:3] offset:64
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[16:19], s[2:3] offset:48
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[24:27], s[2:3] offset:32
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[28:31], s[2:3] offset:16
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[0:3], s[2:3]
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[4:7], s[6:7] offset:112
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[8:11], s[6:7] offset:96
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[12:15], s[6:7] offset:80
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[20:23], s[6:7] offset:64
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[16:19], s[6:7] offset:48
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[24:27], s[6:7] offset:32
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[28:31], s[6:7] offset:16
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v32, v[0:3], s[6:7]
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
@@ -385,12 +385,12 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VALU_WRITE(ptr addrspace(1) noalias %in, ptr addrspace(1) noalias %out) #0 {
; GCN-LABEL: test_sched_group_barrier_pipeline_alternating_READ_VALU_WRITE:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GCN-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GCN-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GCN-NEXT: v_lshlrev_b32_e32 v16, 7, v0
-; GCN-NEXT: ; kill: killed $sgpr0_sgpr1
+; GCN-NEXT: ; kill: killed $sgpr4_sgpr5
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:32
+; GCN-NEXT: global_load_dwordx4 v[12:15], v16, s[4:5] offset:32
; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; GCN-NEXT: s_waitcnt vmcnt(0)
@@ -398,8 +398,8 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; GCN-NEXT: v_mul_lo_u32 v12, v12, v12
; GCN-NEXT: v_mul_lo_u32 v15, v15, v15
; GCN-NEXT: v_mul_lo_u32 v14, v14, v14
-; GCN-NEXT: global_store_dwordx4 v16, v[12:15], s[2:3] offset:32
-; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]
+; GCN-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:32
+; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[4:5]
; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; GCN-NEXT: s_waitcnt vmcnt(0)
@@ -407,30 +407,30 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; GCN-NEXT: v_mul_lo_u32 v2, v2, v2
; GCN-NEXT: v_mul_lo_u32 v1, v1, v1
; GCN-NEXT: v_mul_lo_u32 v0, v0, v0
-; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3]
-; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1] offset:112
+; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]
+; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[4:5] offset:112
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_mul_lo_u32 v3, v3, v3
; GCN-NEXT: v_mul_lo_u32 v2, v2, v2
; GCN-NEXT: v_mul_lo_u32 v1, v1, v1
; GCN-NEXT: v_mul_lo_u32 v0, v0, v0
-; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3] offset:112
-; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1] offset:96
+; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7] offset:112
+; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[4:5] offset:96
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_mul_lo_u32 v3, v3, v3
; GCN-NEXT: v_mul_lo_u32 v2, v2, v2
; GCN-NEXT: v_mul_lo_u32 v1, v1, v1
; GCN-NEXT: v_mul_lo_u32 v0, v0, v0
-; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3] offset:96
-; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1] offset:80
+; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7] offset:96
+; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[4:5] offset:80
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_mul_lo_u32 v3, v3, v3
; GCN-NEXT: v_mul_lo_u32 v2, v2, v2
; GCN-NEXT: v_mul_lo_u32 v1, v1, v1
; GCN-NEXT: v_mul_lo_u32 v0, v0, v0
-; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3] offset:80
-; GCN-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:48
+; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7] offset:80
+; GCN-NEXT: global_load_dwordx4 v[4:7], v16, s[4:5] offset:48
; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
@@ -442,8 +442,8 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; GCN-NEXT: v_mul_lo_u32 v6, v6, v6
; GCN-NEXT: v_mul_lo_u32 v5, v5, v5
; GCN-NEXT: v_mul_lo_u32 v4, v4, v4
-; GCN-NEXT: global_store_dwordx4 v16, v[4:7], s[2:3] offset:48
-; GCN-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:16
+; GCN-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:48
+; GCN-NEXT: global_load_dwordx4 v[8:11], v16, s[4:5] offset:16
; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
@@ -455,8 +455,8 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; GCN-NEXT: v_mul_lo_u32 v8, v8, v8
; GCN-NEXT: v_mul_lo_u32 v11, v11, v11
; GCN-NEXT: v_mul_lo_u32 v10, v10, v10
-; GCN-NEXT: global_store_dwordx4 v16, v[8:11], s[2:3] offset:16
-; GCN-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:64
+; GCN-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:16
+; GCN-NEXT: global_load_dwordx4 v[8:11], v16, s[4:5] offset:64
; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
@@ -468,18 +468,18 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; GCN-NEXT: v_mul_lo_u32 v10, v10, v10
; GCN-NEXT: v_mul_lo_u32 v9, v9, v9
; GCN-NEXT: v_mul_lo_u32 v8, v8, v8
-; GCN-NEXT: global_store_dwordx4 v16, v[8:11], s[2:3] offset:64
+; GCN-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:64
; GCN-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; GCN-NEXT: s_endpgm
;
; EXACTCUTOFF-LABEL: test_sched_group_barrier_pipeline_alternating_READ_VALU_WRITE:
; EXACTCUTOFF: ; %bb.0:
-; EXACTCUTOFF-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; EXACTCUTOFF-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; EXACTCUTOFF-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; EXACTCUTOFF-NEXT: v_lshlrev_b32_e32 v16, 7, v0
-; EXACTCUTOFF-NEXT: ; kill: killed $sgpr0_sgpr1
+; EXACTCUTOFF-NEXT: ; kill: killed $sgpr4_sgpr5
; EXACTCUTOFF-NEXT: s_waitcnt lgkmcnt(0)
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:32
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[12:15], v16, s[4:5] offset:32
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
@@ -487,8 +487,8 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v12, v12, v12
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v15, v15, v15
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v14, v14, v14
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[12:15], s[2:3] offset:32
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:32
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[0:3], v16, s[4:5]
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
@@ -496,30 +496,30 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v2, v2, v2
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v1, v1, v1
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v0, v0, v0
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3]
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1] offset:112
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[0:3], v16, s[4:5] offset:112
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v3, v3, v3
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v2, v2, v2
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v1, v1, v1
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v0, v0, v0
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3] offset:112
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1] offset:96
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7] offset:112
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[0:3], v16, s[4:5] offset:96
; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v3, v3, v3
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v2, v2, v2
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v1, v1, v1
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v0, v0, v0
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3] offset:96
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1] offset:80
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7] offset:96
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[0:3], v16, s[4:5] offset:80
; EXACTCUTOFF-NEXT: s_waitcnt vmcnt(0)
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v3, v3, v3
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v2, v2, v2
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v1, v1, v1
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v0, v0, v0
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3] offset:80
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:48
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7] offset:80
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[4:7], v16, s[4:5] offset:48
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
@@ -531,8 +531,8 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v6, v6, v6
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v5, v5, v5
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v4, v4, v4
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[4:7], s[2:3] offset:48
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:16
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:48
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[8:11], v16, s[4:5] offset:16
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
@@ -544,8 +544,8 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v8, v8, v8
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v11, v11, v11
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v10, v10, v10
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[8:11], s[2:3] offset:16
-; EXACTCUTOFF-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:64
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:16
+; EXACTCUTOFF-NEXT: global_load_dwordx4 v[8:11], v16, s[4:5] offset:64
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000020) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000002) size(2) SyncID(0)
@@ -557,7 +557,7 @@ define amdgpu_kernel void @test_sched_group_barrier_pipeline_alternating_READ_VA
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v10, v10, v10
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v9, v9, v9
; EXACTCUTOFF-NEXT: v_mul_lo_u32 v8, v8, v8
-; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[8:11], s[2:3] offset:64
+; EXACTCUTOFF-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:64
; EXACTCUTOFF-NEXT: ; sched_group_barrier mask(0x00000040) size(1) SyncID(0)
; EXACTCUTOFF-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x() #2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp.ll
index 142145098df87..4bed23487445a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp.ll
@@ -442,59 +442,59 @@ define amdgpu_kernel void @s_exp_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
;
; GFX900-SDAG-LABEL: s_exp_v2f32:
; GFX900-SDAG: ; %bb.0:
-; GFX900-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX900-SDAG-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0x3fb8aa3b
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x32a5705f
; GFX900-SDAG-NEXT: v_mov_b32_e32 v5, 0xc2ce8ed0
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-SDAG-NEXT: v_mul_f32_e32 v2, s3, v0
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v2, s7, v0
; GFX900-SDAG-NEXT: v_rndne_f32_e32 v3, v2
-; GFX900-SDAG-NEXT: v_fma_f32 v4, s3, v0, -v2
+; GFX900-SDAG-NEXT: v_fma_f32 v4, s7, v0, -v2
; GFX900-SDAG-NEXT: v_sub_f32_e32 v2, v2, v3
-; GFX900-SDAG-NEXT: v_fma_f32 v4, s3, v1, v4
-; GFX900-SDAG-NEXT: v_mul_f32_e32 v6, s2, v0
+; GFX900-SDAG-NEXT: v_fma_f32 v4, s7, v1, v4
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v6, s6, v0
; GFX900-SDAG-NEXT: v_add_f32_e32 v2, v2, v4
; GFX900-SDAG-NEXT: v_rndne_f32_e32 v7, v6
-; GFX900-SDAG-NEXT: v_fma_f32 v0, s2, v0, -v6
+; GFX900-SDAG-NEXT: v_fma_f32 v0, s6, v0, -v6
; GFX900-SDAG-NEXT: v_cvt_i32_f32_e32 v3, v3
; GFX900-SDAG-NEXT: v_exp_f32_e32 v2, v2
; GFX900-SDAG-NEXT: v_sub_f32_e32 v8, v6, v7
-; GFX900-SDAG-NEXT: v_fma_f32 v0, s2, v1, v0
+; GFX900-SDAG-NEXT: v_fma_f32 v0, s6, v1, v0
; GFX900-SDAG-NEXT: v_add_f32_e32 v0, v8, v0
; GFX900-SDAG-NEXT: v_exp_f32_e32 v0, v0
; GFX900-SDAG-NEXT: v_cvt_i32_f32_e32 v6, v7
; GFX900-SDAG-NEXT: v_ldexp_f32 v2, v2, v3
-; GFX900-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s3, v5
+; GFX900-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s7, v5
; GFX900-SDAG-NEXT: v_mov_b32_e32 v3, 0x42b17218
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
; GFX900-SDAG-NEXT: v_mov_b32_e32 v7, 0x7f800000
-; GFX900-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s3, v3
+; GFX900-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s7, v3
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc
; GFX900-SDAG-NEXT: v_ldexp_f32 v0, v0, v6
-; GFX900-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v5
+; GFX900-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s6, v5
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX900-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v3
+; GFX900-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s6, v3
; GFX900-SDAG-NEXT: v_mov_b32_e32 v4, 0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, v7, v0, vcc
-; GFX900-SDAG-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
+; GFX900-SDAG-NEXT: global_store_dwordx2 v4, v[0:1], s[4:5]
; GFX900-SDAG-NEXT: s_endpgm
;
; GFX900-GISEL-LABEL: s_exp_v2f32:
; GFX900-GISEL: ; %bb.0:
-; GFX900-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX900-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, 0x3fb8aa3b
; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x32a5705f
; GFX900-GISEL-NEXT: v_mov_b32_e32 v6, 0x7f800000
; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-GISEL-NEXT: v_mul_f32_e32 v2, s2, v0
-; GFX900-GISEL-NEXT: v_fma_f32 v3, s2, v0, -v2
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v2, s6, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v3, s6, v0, -v2
; GFX900-GISEL-NEXT: v_rndne_f32_e32 v4, v2
-; GFX900-GISEL-NEXT: v_mul_f32_e32 v5, s3, v0
-; GFX900-GISEL-NEXT: v_fma_f32 v3, s2, v1, v3
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v5, s7, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v3, s6, v1, v3
; GFX900-GISEL-NEXT: v_sub_f32_e32 v2, v2, v4
-; GFX900-GISEL-NEXT: v_fma_f32 v0, s3, v0, -v5
+; GFX900-GISEL-NEXT: v_fma_f32 v0, s7, v0, -v5
; GFX900-GISEL-NEXT: v_add_f32_e32 v2, v2, v3
-; GFX900-GISEL-NEXT: v_fma_f32 v0, s3, v1, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v0, s7, v1, v0
; GFX900-GISEL-NEXT: v_rndne_f32_e32 v1, v5
; GFX900-GISEL-NEXT: v_cvt_i32_f32_e32 v3, v4
; GFX900-GISEL-NEXT: v_exp_f32_e32 v2, v2
@@ -504,18 +504,18 @@ define amdgpu_kernel void @s_exp_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
; GFX900-GISEL-NEXT: v_exp_f32_e32 v5, v0
; GFX900-GISEL-NEXT: v_mov_b32_e32 v4, 0xc2ce8ed0
; GFX900-GISEL-NEXT: v_ldexp_f32 v2, v2, v3
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v4
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s6, v4
; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, 0x42b17218
; GFX900-GISEL-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc
-; GFX900-GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s2, v3
+; GFX900-GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s6, v3
; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v6, vcc
; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v5, v1
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v4
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s7, v4
; GFX900-GISEL-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
-; GFX900-GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s3, v3
+; GFX900-GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s7, v3
; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX900-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX900-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX900-GISEL-NEXT: s_endpgm
;
; SI-SDAG-LABEL: s_exp_v2f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
index 4d981d27c309e..ec7e52532cd32 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
@@ -444,59 +444,59 @@ define amdgpu_kernel void @s_exp10_v2f32(ptr addrspace(1) %out, <2 x float> %in)
;
; GFX900-SDAG-LABEL: s_exp10_v2f32:
; GFX900-SDAG: ; %bb.0:
-; GFX900-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX900-SDAG-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0x40549a78
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x33979a37
; GFX900-SDAG-NEXT: v_mov_b32_e32 v5, 0xc23369f4
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-SDAG-NEXT: v_mul_f32_e32 v2, s3, v0
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v2, s7, v0
; GFX900-SDAG-NEXT: v_rndne_f32_e32 v3, v2
-; GFX900-SDAG-NEXT: v_fma_f32 v4, s3, v0, -v2
+; GFX900-SDAG-NEXT: v_fma_f32 v4, s7, v0, -v2
; GFX900-SDAG-NEXT: v_sub_f32_e32 v2, v2, v3
-; GFX900-SDAG-NEXT: v_fma_f32 v4, s3, v1, v4
-; GFX900-SDAG-NEXT: v_mul_f32_e32 v6, s2, v0
+; GFX900-SDAG-NEXT: v_fma_f32 v4, s7, v1, v4
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v6, s6, v0
; GFX900-SDAG-NEXT: v_add_f32_e32 v2, v2, v4
; GFX900-SDAG-NEXT: v_rndne_f32_e32 v7, v6
-; GFX900-SDAG-NEXT: v_fma_f32 v0, s2, v0, -v6
+; GFX900-SDAG-NEXT: v_fma_f32 v0, s6, v0, -v6
; GFX900-SDAG-NEXT: v_cvt_i32_f32_e32 v3, v3
; GFX900-SDAG-NEXT: v_exp_f32_e32 v2, v2
; GFX900-SDAG-NEXT: v_sub_f32_e32 v8, v6, v7
-; GFX900-SDAG-NEXT: v_fma_f32 v0, s2, v1, v0
+; GFX900-SDAG-NEXT: v_fma_f32 v0, s6, v1, v0
; GFX900-SDAG-NEXT: v_add_f32_e32 v0, v8, v0
; GFX900-SDAG-NEXT: v_exp_f32_e32 v0, v0
; GFX900-SDAG-NEXT: v_cvt_i32_f32_e32 v6, v7
; GFX900-SDAG-NEXT: v_ldexp_f32 v2, v2, v3
-; GFX900-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s3, v5
+; GFX900-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s7, v5
; GFX900-SDAG-NEXT: v_mov_b32_e32 v3, 0x421a209b
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
; GFX900-SDAG-NEXT: v_mov_b32_e32 v7, 0x7f800000
-; GFX900-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s3, v3
+; GFX900-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s7, v3
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc
; GFX900-SDAG-NEXT: v_ldexp_f32 v0, v0, v6
-; GFX900-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v5
+; GFX900-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s6, v5
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX900-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v3
+; GFX900-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s6, v3
; GFX900-SDAG-NEXT: v_mov_b32_e32 v4, 0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, v7, v0, vcc
-; GFX900-SDAG-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
+; GFX900-SDAG-NEXT: global_store_dwordx2 v4, v[0:1], s[4:5]
; GFX900-SDAG-NEXT: s_endpgm
;
; GFX900-GISEL-LABEL: s_exp10_v2f32:
; GFX900-GISEL: ; %bb.0:
-; GFX900-GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX900-GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX900-GISEL-NEXT: v_mov_b32_e32 v0, 0x40549a78
; GFX900-GISEL-NEXT: v_mov_b32_e32 v1, 0x33979a37
; GFX900-GISEL-NEXT: v_mov_b32_e32 v6, 0x7f800000
; GFX900-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-GISEL-NEXT: v_mul_f32_e32 v2, s2, v0
-; GFX900-GISEL-NEXT: v_fma_f32 v3, s2, v0, -v2
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v2, s6, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v3, s6, v0, -v2
; GFX900-GISEL-NEXT: v_rndne_f32_e32 v4, v2
-; GFX900-GISEL-NEXT: v_mul_f32_e32 v5, s3, v0
-; GFX900-GISEL-NEXT: v_fma_f32 v3, s2, v1, v3
+; GFX900-GISEL-NEXT: v_mul_f32_e32 v5, s7, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v3, s6, v1, v3
; GFX900-GISEL-NEXT: v_sub_f32_e32 v2, v2, v4
-; GFX900-GISEL-NEXT: v_fma_f32 v0, s3, v0, -v5
+; GFX900-GISEL-NEXT: v_fma_f32 v0, s7, v0, -v5
; GFX900-GISEL-NEXT: v_add_f32_e32 v2, v2, v3
-; GFX900-GISEL-NEXT: v_fma_f32 v0, s3, v1, v0
+; GFX900-GISEL-NEXT: v_fma_f32 v0, s7, v1, v0
; GFX900-GISEL-NEXT: v_rndne_f32_e32 v1, v5
; GFX900-GISEL-NEXT: v_cvt_i32_f32_e32 v3, v4
; GFX900-GISEL-NEXT: v_exp_f32_e32 v2, v2
@@ -506,18 +506,18 @@ define amdgpu_kernel void @s_exp10_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; GFX900-GISEL-NEXT: v_exp_f32_e32 v5, v0
; GFX900-GISEL-NEXT: v_mov_b32_e32 v4, 0xc23369f4
; GFX900-GISEL-NEXT: v_ldexp_f32 v2, v2, v3
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s2, v4
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s6, v4
; GFX900-GISEL-NEXT: v_mov_b32_e32 v3, 0x421a209b
; GFX900-GISEL-NEXT: v_cndmask_b32_e64 v2, v2, 0, vcc
-; GFX900-GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s2, v3
+; GFX900-GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s6, v3
; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v6, vcc
; GFX900-GISEL-NEXT: v_ldexp_f32 v1, v5, v1
-; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s3, v4
+; GFX900-GISEL-NEXT: v_cmp_lt_f32_e32 vcc, s7, v4
; GFX900-GISEL-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
-; GFX900-GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s3, v3
+; GFX900-GISEL-NEXT: v_cmp_gt_f32_e32 vcc, s7, v3
; GFX900-GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GFX900-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX900-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX900-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX900-GISEL-NEXT: s_endpgm
;
; SI-SDAG-LABEL: s_exp10_v2f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
index 9f80e66e8f873..acbb8684da924 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
@@ -271,25 +271,25 @@ define amdgpu_kernel void @s_exp2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
;
; GFX900-SDAG-LABEL: s_exp2_v2f32:
; GFX900-SDAG: ; %bb.0:
-; GFX900-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX900-SDAG-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0xc2fc0000
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x1f800000
; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0x42800000
; GFX900-SDAG-NEXT: v_mov_b32_e32 v5, 0
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s7, v0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v3, 1.0, v1, vcc
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v2, vcc
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 1.0, v1, vcc
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc
-; GFX900-SDAG-NEXT: v_add_f32_e32 v4, s3, v4
-; GFX900-SDAG-NEXT: v_add_f32_e32 v1, s2, v1
+; GFX900-SDAG-NEXT: v_add_f32_e32 v4, s7, v4
+; GFX900-SDAG-NEXT: v_add_f32_e32 v1, s6, v1
; GFX900-SDAG-NEXT: v_exp_f32_e32 v4, v4
; GFX900-SDAG-NEXT: v_exp_f32_e32 v2, v1
; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, v4, v3
; GFX900-SDAG-NEXT: v_mul_f32_e32 v0, v2, v0
-; GFX900-SDAG-NEXT: global_store_dwordx2 v5, v[0:1], s[0:1]
+; GFX900-SDAG-NEXT: global_store_dwordx2 v5, v[0:1], s[4:5]
; GFX900-SDAG-NEXT: s_endpgm
;
; GFX900-GISEL-LABEL: s_exp2_v2f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
index 7ca04cc235605..bc4d03e7af260 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
@@ -314,25 +314,25 @@ define amdgpu_kernel void @s_log2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
;
; GFX900-SDAG-LABEL: s_log2_v2f32:
; GFX900-SDAG: ; %bb.0:
-; GFX900-SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX900-SDAG-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x42000000
; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0x4f800000
; GFX900-SDAG-NEXT: v_mov_b32_e32 v5, 0
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s7, v0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v3, 0, v1, vcc
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v4, 1.0, v2, vcc
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, 1.0, v2, vcc
-; GFX900-SDAG-NEXT: v_mul_f32_e32 v4, s3, v4
-; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, s2, v1
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v4, s7, v4
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v1, s6, v1
; GFX900-SDAG-NEXT: v_log_f32_e32 v4, v4
; GFX900-SDAG-NEXT: v_log_f32_e32 v2, v1
; GFX900-SDAG-NEXT: v_sub_f32_e32 v1, v4, v3
; GFX900-SDAG-NEXT: v_sub_f32_e32 v0, v2, v0
-; GFX900-SDAG-NEXT: global_store_dwordx2 v5, v[0:1], s[0:1]
+; GFX900-SDAG-NEXT: global_store_dwordx2 v5, v[0:1], s[4:5]
; GFX900-SDAG-NEXT: s_endpgm
;
; GFX900-GISEL-LABEL: s_log2_v2f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.round.ll b/llvm/test/CodeGen/AMDGPU/llvm.round.ll
index d5b4f879bf8a0..9327d76e50692 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.round.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.round.ll
@@ -115,31 +115,57 @@ define amdgpu_kernel void @round_v2f32(ptr addrspace(1) %out, <2 x float> %in) #
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
-; GFX89-LABEL: round_v2f32:
-; GFX89: ; %bb.0:
-; GFX89-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX89-NEXT: s_brev_b32 s8, -2
-; GFX89-NEXT: s_mov_b32 s7, 0xf000
-; GFX89-NEXT: s_mov_b32 s6, -1
-; GFX89-NEXT: s_waitcnt lgkmcnt(0)
-; GFX89-NEXT: v_trunc_f32_e32 v0, s3
-; GFX89-NEXT: v_sub_f32_e32 v1, s3, v0
-; GFX89-NEXT: s_mov_b32 s4, s0
-; GFX89-NEXT: s_mov_b32 s5, s1
-; GFX89-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, 0.5
-; GFX89-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, s[0:1]
-; GFX89-NEXT: v_mov_b32_e32 v2, s3
-; GFX89-NEXT: v_bfi_b32 v1, s8, v1, v2
-; GFX89-NEXT: v_add_f32_e32 v1, v0, v1
-; GFX89-NEXT: v_trunc_f32_e32 v0, s2
-; GFX89-NEXT: v_sub_f32_e32 v2, s2, v0
-; GFX89-NEXT: v_cmp_ge_f32_e64 s[0:1], |v2|, 0.5
-; GFX89-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, s[0:1]
-; GFX89-NEXT: v_mov_b32_e32 v3, s2
-; GFX89-NEXT: v_bfi_b32 v2, s8, v2, v3
-; GFX89-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX89-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
-; GFX89-NEXT: s_endpgm
+; GFX8-LABEL: round_v2f32:
+; GFX8: ; %bb.0:
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX8-NEXT: s_brev_b32 s8, -2
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_trunc_f32_e32 v0, s3
+; GFX8-NEXT: v_sub_f32_e32 v1, s3, v0
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, 0.5
+; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v2, s3
+; GFX8-NEXT: v_bfi_b32 v1, s8, v1, v2
+; GFX8-NEXT: v_add_f32_e32 v1, v0, v1
+; GFX8-NEXT: v_trunc_f32_e32 v0, s2
+; GFX8-NEXT: v_sub_f32_e32 v2, s2, v0
+; GFX8-NEXT: v_cmp_ge_f32_e64 s[0:1], |v2|, 0.5
+; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v3, s2
+; GFX8-NEXT: v_bfi_b32 v2, s8, v2, v3
+; GFX8-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX8-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX8-NEXT: s_endpgm
+;
+; GFX9-LABEL: round_v2f32:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX9-NEXT: s_brev_b32 s8, -2
+; GFX9-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-NEXT: s_mov_b32 s2, -1
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_trunc_f32_e32 v0, s7
+; GFX9-NEXT: v_sub_f32_e32 v1, s7, v0
+; GFX9-NEXT: s_mov_b32 s0, s4
+; GFX9-NEXT: s_mov_b32 s1, s5
+; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, 0.5
+; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, s[4:5]
+; GFX9-NEXT: v_mov_b32_e32 v2, s7
+; GFX9-NEXT: v_bfi_b32 v1, s8, v1, v2
+; GFX9-NEXT: v_add_f32_e32 v1, v0, v1
+; GFX9-NEXT: v_trunc_f32_e32 v0, s6
+; GFX9-NEXT: v_sub_f32_e32 v2, s6, v0
+; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, 0.5
+; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, s[4:5]
+; GFX9-NEXT: v_mov_b32_e32 v3, s6
+; GFX9-NEXT: v_bfi_b32 v2, s8, v2, v3
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: round_v2f32:
; GFX11: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll b/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll
index e8ac1b2887c36..48df9a0d98d6b 100644
--- a/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll
@@ -8,12 +8,12 @@
define amdgpu_kernel void @s_lshr_v2i16(ptr addrspace(1) %out, <2 x i16> %lhs, <2 x i16> %rhs) #0 {
; GFX9-LABEL: s_lshr_v2i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: v_pk_lshrrev_b16 v1, s3, v1
-; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: v_pk_lshrrev_b16 v1, s7, v1
+; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: s_endpgm
;
; VI-LABEL: s_lshr_v2i16:
@@ -54,11 +54,11 @@ define amdgpu_kernel void @s_lshr_v2i16(ptr addrspace(1) %out, <2 x i16> %lhs, <
;
; GFX10-LABEL: s_lshr_v2i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_pk_lshrrev_b16 v1, s3, s2
-; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: v_pk_lshrrev_b16 v1, s7, s6
+; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_lshr_v2i16:
diff --git a/llvm/test/CodeGen/AMDGPU/madak.ll b/llvm/test/CodeGen/AMDGPU/madak.ll
index b8b4d4440d580..f2815915b8425 100644
--- a/llvm/test/CodeGen/AMDGPU/madak.ll
+++ b/llvm/test/CodeGen/AMDGPU/madak.ll
@@ -225,40 +225,40 @@ define amdgpu_kernel void @madak_2_use_f32(ptr addrspace(1) noalias %out, ptr ad
;
; GFX9-LABEL: madak_2_use_f32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v4, 0x41200000
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v0, s[2:3] glc
+; GFX9-NEXT: global_load_dword v1, v0, s[6:7] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: global_load_dword v2, v0, s[2:3] offset:4 glc
+; GFX9-NEXT: global_load_dword v2, v0, s[6:7] offset:4 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: global_load_dword v3, v0, s[2:3] offset:8 glc
+; GFX9-NEXT: global_load_dword v3, v0, s[6:7] offset:8 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_madak_f32 v2, v1, v2, 0x41200000
; GFX9-NEXT: v_mac_f32_e32 v4, v1, v3
-; GFX9-NEXT: global_store_dword v0, v2, s[0:1]
+; GFX9-NEXT: global_store_dword v0, v2, s[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: global_store_dword v0, v4, s[2:3] offset:4
+; GFX9-NEXT: global_store_dword v0, v4, s[6:7] offset:4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_endpgm
;
; GFX10-MAD-LABEL: madak_2_use_f32:
; GFX10-MAD: ; %bb.0:
-; GFX10-MAD-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-MAD-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-MAD-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-MAD-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-MAD-NEXT: global_load_dword v1, v0, s[2:3] glc dlc
+; GFX10-MAD-NEXT: global_load_dword v1, v0, s[6:7] glc dlc
; GFX10-MAD-NEXT: s_waitcnt vmcnt(0)
-; GFX10-MAD-NEXT: global_load_dword v2, v0, s[2:3] offset:4 glc dlc
+; GFX10-MAD-NEXT: global_load_dword v2, v0, s[6:7] offset:4 glc dlc
; GFX10-MAD-NEXT: s_waitcnt vmcnt(0)
-; GFX10-MAD-NEXT: global_load_dword v3, v0, s[2:3] offset:8 glc dlc
+; GFX10-MAD-NEXT: global_load_dword v3, v0, s[6:7] offset:8 glc dlc
; GFX10-MAD-NEXT: s_waitcnt vmcnt(0)
; GFX10-MAD-NEXT: v_madak_f32 v2, v1, v2, 0x41200000
; GFX10-MAD-NEXT: v_madak_f32 v1, v1, v3, 0x41200000
-; GFX10-MAD-NEXT: global_store_dword v0, v2, s[0:1]
+; GFX10-MAD-NEXT: global_store_dword v0, v2, s[4:5]
; GFX10-MAD-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-MAD-NEXT: global_store_dword v0, v1, s[2:3] offset:4
+; GFX10-MAD-NEXT: global_store_dword v0, v1, s[6:7] offset:4
; GFX10-MAD-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-MAD-NEXT: s_endpgm
;
@@ -289,41 +289,41 @@ define amdgpu_kernel void @madak_2_use_f32(ptr addrspace(1) noalias %out, ptr ad
;
; GFX940-FMA-LABEL: madak_2_use_f32:
; GFX940-FMA: ; %bb.0:
-; GFX940-FMA-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX940-FMA-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX940-FMA-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX940-FMA-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX940-FMA-NEXT: v_mov_b32_e32 v4, 0x41200000
; GFX940-FMA-NEXT: s_waitcnt lgkmcnt(0)
-; GFX940-FMA-NEXT: global_load_dword v1, v0, s[2:3] sc0 sc1
+; GFX940-FMA-NEXT: global_load_dword v1, v0, s[6:7] sc0 sc1
; GFX940-FMA-NEXT: s_waitcnt vmcnt(0)
-; GFX940-FMA-NEXT: global_load_dword v2, v0, s[2:3] offset:4 sc0 sc1
+; GFX940-FMA-NEXT: global_load_dword v2, v0, s[6:7] offset:4 sc0 sc1
; GFX940-FMA-NEXT: s_waitcnt vmcnt(0)
-; GFX940-FMA-NEXT: global_load_dword v3, v0, s[2:3] offset:8 sc0 sc1
+; GFX940-FMA-NEXT: global_load_dword v3, v0, s[6:7] offset:8 sc0 sc1
; GFX940-FMA-NEXT: s_waitcnt vmcnt(0)
; GFX940-FMA-NEXT: v_fmaak_f32 v2, v1, v2, 0x41200000
; GFX940-FMA-NEXT: v_fmac_f32_e32 v4, v1, v3
-; GFX940-FMA-NEXT: global_store_dword v0, v2, s[0:1] sc0 sc1
+; GFX940-FMA-NEXT: global_store_dword v0, v2, s[4:5] sc0 sc1
; GFX940-FMA-NEXT: s_waitcnt vmcnt(0)
-; GFX940-FMA-NEXT: global_store_dword v0, v4, s[2:3] offset:4 sc0 sc1
+; GFX940-FMA-NEXT: global_store_dword v0, v4, s[6:7] offset:4 sc0 sc1
; GFX940-FMA-NEXT: s_waitcnt vmcnt(0)
; GFX940-FMA-NEXT: s_endpgm
;
; GFX10-FMA-LABEL: madak_2_use_f32:
; GFX10-FMA: ; %bb.0:
-; GFX10-FMA-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-FMA-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-FMA-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-FMA-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-FMA-NEXT: global_load_dword v1, v0, s[2:3] glc dlc
+; GFX10-FMA-NEXT: global_load_dword v1, v0, s[6:7] glc dlc
; GFX10-FMA-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FMA-NEXT: global_load_dword v2, v0, s[2:3] offset:4 glc dlc
+; GFX10-FMA-NEXT: global_load_dword v2, v0, s[6:7] offset:4 glc dlc
; GFX10-FMA-NEXT: s_waitcnt vmcnt(0)
-; GFX10-FMA-NEXT: global_load_dword v3, v0, s[2:3] offset:8 glc dlc
+; GFX10-FMA-NEXT: global_load_dword v3, v0, s[6:7] offset:8 glc dlc
; GFX10-FMA-NEXT: s_waitcnt vmcnt(0)
; GFX10-FMA-NEXT: v_fmaak_f32 v2, v1, v2, 0x41200000
; GFX10-FMA-NEXT: v_fmaak_f32 v1, v1, v3, 0x41200000
-; GFX10-FMA-NEXT: global_store_dword v0, v2, s[0:1]
+; GFX10-FMA-NEXT: global_store_dword v0, v2, s[4:5]
; GFX10-FMA-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-FMA-NEXT: global_store_dword v0, v1, s[2:3] offset:4
+; GFX10-FMA-NEXT: global_store_dword v0, v1, s[6:7] offset:4
; GFX10-FMA-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-FMA-NEXT: s_endpgm
;
@@ -408,24 +408,24 @@ define amdgpu_kernel void @madak_m_inline_imm_f32(ptr addrspace(1) noalias %out,
;
; GFX9-LABEL: madak_m_inline_imm_f32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v0, s[2:3]
+; GFX9-NEXT: global_load_dword v1, v0, s[6:7]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_madak_f32 v1, 4.0, v1, 0x41200000
-; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX10-MAD-LABEL: madak_m_inline_imm_f32:
; GFX10-MAD: ; %bb.0:
-; GFX10-MAD-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-MAD-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-MAD-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-MAD-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-MAD-NEXT: global_load_dword v1, v0, s[2:3]
+; GFX10-MAD-NEXT: global_load_dword v1, v0, s[6:7]
; GFX10-MAD-NEXT: s_waitcnt vmcnt(0)
; GFX10-MAD-NEXT: v_madak_f32 v1, 4.0, v1, 0x41200000
-; GFX10-MAD-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-MAD-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-MAD-NEXT: s_endpgm
;
; GFX11-MAD-LABEL: madak_m_inline_imm_f32:
@@ -446,25 +446,25 @@ define amdgpu_kernel void @madak_m_inline_imm_f32(ptr addrspace(1) noalias %out,
;
; GFX940-FMA-LABEL: madak_m_inline_imm_f32:
; GFX940-FMA: ; %bb.0:
-; GFX940-FMA-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX940-FMA-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX940-FMA-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX940-FMA-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX940-FMA-NEXT: s_waitcnt lgkmcnt(0)
-; GFX940-FMA-NEXT: global_load_dword v1, v0, s[2:3]
+; GFX940-FMA-NEXT: global_load_dword v1, v0, s[6:7]
; GFX940-FMA-NEXT: s_waitcnt vmcnt(0)
; GFX940-FMA-NEXT: v_fmaak_f32 v1, 4.0, v1, 0x41200000
-; GFX940-FMA-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
+; GFX940-FMA-NEXT: global_store_dword v0, v1, s[4:5] sc0 sc1
; GFX940-FMA-NEXT: s_endpgm
;
; GFX10-FMA-LABEL: madak_m_inline_imm_f32:
; GFX10-FMA: ; %bb.0:
-; GFX10-FMA-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-FMA-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-FMA-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-FMA-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-FMA-NEXT: global_load_dword v1, v0, s[2:3]
+; GFX10-FMA-NEXT: global_load_dword v1, v0, s[6:7]
; GFX10-FMA-NEXT: s_waitcnt vmcnt(0)
; GFX10-FMA-NEXT: v_fmaak_f32 v1, 4.0, v1, 0x41200000
-; GFX10-FMA-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-FMA-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-FMA-NEXT: s_endpgm
;
; GFX11-FMA-LABEL: madak_m_inline_imm_f32:
@@ -961,23 +961,23 @@ define amdgpu_kernel void @s_s_madak_f32(ptr addrspace(1) %out, float %a, float
;
; GFX9-LABEL: s_s_madak_f32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0x41200000
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, s3
-; GFX9-NEXT: v_mac_f32_e32 v1, s2, v2
-; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v2, s7
+; GFX9-NEXT: v_mac_f32_e32 v1, s6, v2
+; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX10-MAD-LABEL: s_s_madak_f32:
; GFX10-MAD: ; %bb.0:
-; GFX10-MAD-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-MAD-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-MAD-NEXT: v_mov_b32_e32 v1, 0
; GFX10-MAD-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-MAD-NEXT: v_mov_b32_e32 v0, s3
-; GFX10-MAD-NEXT: v_madak_f32 v0, s2, v0, 0x41200000
-; GFX10-MAD-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-MAD-NEXT: v_mov_b32_e32 v0, s7
+; GFX10-MAD-NEXT: v_madak_f32 v0, s6, v0, 0x41200000
+; GFX10-MAD-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-MAD-NEXT: s_endpgm
;
; GFX11-MAD-LABEL: s_s_madak_f32:
@@ -994,23 +994,23 @@ define amdgpu_kernel void @s_s_madak_f32(ptr addrspace(1) %out, float %a, float
;
; GFX940-FMA-LABEL: s_s_madak_f32:
; GFX940-FMA: ; %bb.0:
-; GFX940-FMA-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX940-FMA-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX940-FMA-NEXT: v_mov_b32_e32 v1, 0x41200000
; GFX940-FMA-NEXT: v_mov_b32_e32 v0, 0
; GFX940-FMA-NEXT: s_waitcnt lgkmcnt(0)
-; GFX940-FMA-NEXT: v_mov_b32_e32 v2, s3
-; GFX940-FMA-NEXT: v_fmac_f32_e32 v1, s2, v2
-; GFX940-FMA-NEXT: global_store_dword v0, v1, s[0:1] sc0 sc1
+; GFX940-FMA-NEXT: v_mov_b32_e32 v2, s7
+; GFX940-FMA-NEXT: v_fmac_f32_e32 v1, s6, v2
+; GFX940-FMA-NEXT: global_store_dword v0, v1, s[4:5] sc0 sc1
; GFX940-FMA-NEXT: s_endpgm
;
; GFX10-FMA-LABEL: s_s_madak_f32:
; GFX10-FMA: ; %bb.0:
-; GFX10-FMA-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-FMA-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-FMA-NEXT: v_mov_b32_e32 v1, 0
; GFX10-FMA-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-FMA-NEXT: v_mov_b32_e32 v0, s3
-; GFX10-FMA-NEXT: v_fmaak_f32 v0, s2, v0, 0x41200000
-; GFX10-FMA-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-FMA-NEXT: v_mov_b32_e32 v0, s7
+; GFX10-FMA-NEXT: v_fmaak_f32 v0, s6, v0, 0x41200000
+; GFX10-FMA-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-FMA-NEXT: s_endpgm
;
; GFX11-FMA-LABEL: s_s_madak_f32:
diff --git a/llvm/test/CodeGen/AMDGPU/memory_clause.ll b/llvm/test/CodeGen/AMDGPU/memory_clause.ll
index 3a065d518f0a9..1de9206801e2a 100644
--- a/llvm/test/CodeGen/AMDGPU/memory_clause.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory_clause.ll
@@ -5,21 +5,21 @@
define amdgpu_kernel void @vector_clause(ptr addrspace(1) noalias nocapture readonly %arg, ptr addrspace(1) noalias nocapture %arg1) {
; GCN-LABEL: vector_clause:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GCN-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GCN-NEXT: v_lshlrev_b32_e32 v16, 4, v0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[0:1]
-; GCN-NEXT: global_load_dwordx4 v[4:7], v16, s[0:1] offset:16
-; GCN-NEXT: global_load_dwordx4 v[8:11], v16, s[0:1] offset:32
-; GCN-NEXT: global_load_dwordx4 v[12:15], v16, s[0:1] offset:48
+; GCN-NEXT: global_load_dwordx4 v[0:3], v16, s[4:5]
+; GCN-NEXT: global_load_dwordx4 v[4:7], v16, s[4:5] offset:16
+; GCN-NEXT: global_load_dwordx4 v[8:11], v16, s[4:5] offset:32
+; GCN-NEXT: global_load_dwordx4 v[12:15], v16, s[4:5] offset:48
; GCN-NEXT: s_waitcnt vmcnt(3)
-; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[2:3]
+; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[6:7]
; GCN-NEXT: s_waitcnt vmcnt(3)
-; GCN-NEXT: global_store_dwordx4 v16, v[4:7], s[2:3] offset:16
+; GCN-NEXT: global_store_dwordx4 v16, v[4:7], s[6:7] offset:16
; GCN-NEXT: s_waitcnt vmcnt(3)
-; GCN-NEXT: global_store_dwordx4 v16, v[8:11], s[2:3] offset:32
+; GCN-NEXT: global_store_dwordx4 v16, v[8:11], s[6:7] offset:32
; GCN-NEXT: s_waitcnt vmcnt(3)
-; GCN-NEXT: global_store_dwordx4 v16, v[12:15], s[2:3] offset:48
+; GCN-NEXT: global_store_dwordx4 v16, v[12:15], s[6:7] offset:48
; GCN-NEXT: s_endpgm
;
; GCN-SCRATCH-LABEL: vector_clause:
diff --git a/llvm/test/CodeGen/AMDGPU/merge-s-load.mir b/llvm/test/CodeGen/AMDGPU/merge-s-load.mir
index b08da2e1848ff..28d30a70e50c1 100644
--- a/llvm/test/CodeGen/AMDGPU/merge-s-load.mir
+++ b/llvm/test/CodeGen/AMDGPU/merge-s-load.mir
@@ -1,4 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1013 -verify-machineinstrs -run-pass si-load-store-opt -o - %s | FileCheck %s -check-prefixes=CHECK,GFX10
# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -verify-machineinstrs -run-pass si-load-store-opt -o - %s | FileCheck %s -check-prefixes=CHECK,GFX11
# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass si-load-store-opt -o - %s | FileCheck %s -check-prefixes=CHECK,GFX12
@@ -6,11 +7,23 @@
name: merge_s_load_x1_x1
body: |
bb.0:
- ; CHECK-LABEL: name: merge_s_load_x1_x1
- ; CHECK: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
- ; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s64), align 4)
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[S_LOAD_DWORDX2_IMM]].sub0
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sreg_32_xm0_xexec = COPY killed [[S_LOAD_DWORDX2_IMM]].sub1
+ ; GFX10-LABEL: name: merge_s_load_x1_x1
+ ; GFX10: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX10-NEXT: early-clobber %3:sreg_64_xexec = S_LOAD_DWORDX2_IMM_ec [[DEF]], 0, 0 :: (dereferenceable invariant load (s64), align 4)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:sreg_32_xm0_xexec = COPY %3.sub0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:sreg_32_xm0_xexec = COPY killed %3.sub1
+ ;
+ ; GFX11-LABEL: name: merge_s_load_x1_x1
+ ; GFX11: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s64), align 4)
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[S_LOAD_DWORDX2_IMM]].sub0
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sreg_32_xm0_xexec = COPY killed [[S_LOAD_DWORDX2_IMM]].sub1
+ ;
+ ; GFX12-LABEL: name: merge_s_load_x1_x1
+ ; GFX12: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s64), align 4)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[S_LOAD_DWORDX2_IMM]].sub0
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32_xm0_xexec = COPY killed [[S_LOAD_DWORDX2_IMM]].sub1
%0:sgpr_64 = IMPLICIT_DEF
%1:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0:sgpr_64, 0, 0 :: (dereferenceable invariant load (s32))
%2:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0:sgpr_64, 4, 0 :: (dereferenceable invariant load (s32))
@@ -46,6 +59,13 @@ body: |
name: merge_s_load_x1_x1_x1
body: |
bb.0:
+ ; GFX10-LABEL: name: merge_s_load_x1_x1_x1
+ ; GFX10: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX10-NEXT: early-clobber %4:sreg_64_xexec = S_LOAD_DWORDX2_IMM_ec [[DEF]], 0, 0 :: (dereferenceable invariant load (s64), align 4)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:sreg_32_xm0_xexec = COPY %4.sub0
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:sreg_32_xm0_xexec = COPY killed %4.sub1
+ ; GFX10-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[DEF]], 8, 0 :: (dereferenceable invariant load (s32))
+ ;
; GFX11-LABEL: name: merge_s_load_x1_x1_x1
; GFX11: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
; GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s64), align 4)
@@ -70,6 +90,16 @@ body: |
name: merge_s_load_x1_x1_x1_x1
body: |
bb.0:
+ ; GFX10-LABEL: name: merge_s_load_x1_x1_x1_x1
+ ; GFX10: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX10-NEXT: early-clobber %7:sgpr_128 = S_LOAD_DWORDX4_IMM_ec [[DEF]], 0, 0 :: (dereferenceable invariant load (s128), align 4)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:sreg_64_xexec = COPY %7.sub0_sub1
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:sreg_64_xexec = COPY killed %7.sub2_sub3
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[COPY]].sub0
+ ; GFX10-NEXT: [[COPY3:%[0-9]+]]:sreg_32_xm0_xexec = COPY killed [[COPY]].sub1
+ ; GFX10-NEXT: [[COPY4:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[COPY1]].sub0
+ ; GFX10-NEXT: [[COPY5:%[0-9]+]]:sreg_32_xm0_xexec = COPY killed [[COPY1]].sub1
+ ;
; GFX11-LABEL: name: merge_s_load_x1_x1_x1_x1
; GFX11: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
; GFX11-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s128), align 4)
@@ -100,6 +130,24 @@ body: |
name: merge_s_load_x1_x1_x1_x1_x1_x1_x1_x1
body: |
bb.0:
+ ; GFX10-LABEL: name: merge_s_load_x1_x1_x1_x1_x1_x1_x1_x1
+ ; GFX10: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX10-NEXT: early-clobber %15:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[DEF]], 0, 0 :: (dereferenceable invariant load (s256), align 4)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr_128 = COPY %15.sub0_sub1_sub2_sub3
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:sgpr_128 = COPY killed %15.sub4_sub5_sub6_sub7
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:sreg_64_xexec = COPY [[COPY]].sub0_sub1
+ ; GFX10-NEXT: [[COPY3:%[0-9]+]]:sreg_64_xexec = COPY killed [[COPY]].sub2_sub3
+ ; GFX10-NEXT: [[COPY4:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[COPY2]].sub0
+ ; GFX10-NEXT: [[COPY5:%[0-9]+]]:sreg_32_xm0_xexec = COPY killed [[COPY2]].sub1
+ ; GFX10-NEXT: [[COPY6:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[COPY3]].sub0
+ ; GFX10-NEXT: [[COPY7:%[0-9]+]]:sreg_32_xm0_xexec = COPY killed [[COPY3]].sub1
+ ; GFX10-NEXT: [[COPY8:%[0-9]+]]:sreg_64_xexec = COPY [[COPY1]].sub0_sub1
+ ; GFX10-NEXT: [[COPY9:%[0-9]+]]:sreg_64_xexec = COPY killed [[COPY1]].sub2_sub3
+ ; GFX10-NEXT: [[COPY10:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[COPY8]].sub0
+ ; GFX10-NEXT: [[COPY11:%[0-9]+]]:sreg_32_xm0_xexec = COPY killed [[COPY8]].sub1
+ ; GFX10-NEXT: [[COPY12:%[0-9]+]]:sreg_32_xm0_xexec = COPY [[COPY9]].sub0
+ ; GFX10-NEXT: [[COPY13:%[0-9]+]]:sreg_32_xm0_xexec = COPY killed [[COPY9]].sub1
+ ;
; GFX11-LABEL: name: merge_s_load_x1_x1_x1_x1_x1_x1_x1_x1
; GFX11: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
; GFX11-NEXT: [[S_LOAD_DWORDX8_IMM:%[0-9]+]]:sgpr_256 = S_LOAD_DWORDX8_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s256), align 4)
@@ -150,6 +198,11 @@ body: |
name: merge_s_load_x2_x1
body: |
bb.0:
+ ; GFX10-LABEL: name: merge_s_load_x2_x1
+ ; GFX10: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX10-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sgpr_64 = S_LOAD_DWORDX2_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s64))
+ ; GFX10-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[DEF]], 8, 0 :: (dereferenceable invariant load (s32))
+ ;
; GFX11-LABEL: name: merge_s_load_x2_x1
; GFX11: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
; GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sgpr_64 = S_LOAD_DWORDX2_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s64))
@@ -169,11 +222,23 @@ body: |
name: merge_s_load_x2_x2
body: |
bb.0:
- ; CHECK-LABEL: name: merge_s_load_x2_x2
- ; CHECK: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s128), align 8)
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64 = COPY [[S_LOAD_DWORDX4_IMM]].sub0_sub1
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr_64 = COPY killed [[S_LOAD_DWORDX4_IMM]].sub2_sub3
+ ; GFX10-LABEL: name: merge_s_load_x2_x2
+ ; GFX10: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX10-NEXT: early-clobber %3:sgpr_128 = S_LOAD_DWORDX4_IMM_ec [[DEF]], 0, 0 :: (dereferenceable invariant load (s128), align 8)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr_64 = COPY %3.sub0_sub1
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:sgpr_64 = COPY killed %3.sub2_sub3
+ ;
+ ; GFX11-LABEL: name: merge_s_load_x2_x2
+ ; GFX11: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX11-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s128), align 8)
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_64 = COPY [[S_LOAD_DWORDX4_IMM]].sub0_sub1
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_64 = COPY killed [[S_LOAD_DWORDX4_IMM]].sub2_sub3
+ ;
+ ; GFX12-LABEL: name: merge_s_load_x2_x2
+ ; GFX12: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s128), align 8)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sgpr_64 = COPY [[S_LOAD_DWORDX4_IMM]].sub0_sub1
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_64 = COPY killed [[S_LOAD_DWORDX4_IMM]].sub2_sub3
%0:sgpr_64 = IMPLICIT_DEF
%1:sgpr_64 = S_LOAD_DWORDX2_IMM %0:sgpr_64, 0, 0 :: (dereferenceable invariant load (s64))
%2:sgpr_64 = S_LOAD_DWORDX2_IMM %0:sgpr_64, 8, 0 :: (dereferenceable invariant load (s64))
@@ -183,15 +248,35 @@ body: |
name: merge_s_load_x2_x2_x2_x2
body: |
bb.0:
- ; CHECK-LABEL: name: merge_s_load_x2_x2_x2_x2
- ; CHECK: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
- ; CHECK-NEXT: [[S_LOAD_DWORDX8_IMM:%[0-9]+]]:sgpr_256 = S_LOAD_DWORDX8_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s256), align 8)
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_128 = COPY [[S_LOAD_DWORDX8_IMM]].sub0_sub1_sub2_sub3
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr_128 = COPY killed [[S_LOAD_DWORDX8_IMM]].sub4_sub5_sub6_sub7
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr_64 = COPY [[COPY]].sub0_sub1
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:sgpr_64 = COPY killed [[COPY]].sub2_sub3
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:sgpr_64 = COPY [[COPY1]].sub0_sub1
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:sgpr_64 = COPY killed [[COPY1]].sub2_sub3
+ ; GFX10-LABEL: name: merge_s_load_x2_x2_x2_x2
+ ; GFX10: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX10-NEXT: early-clobber %7:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[DEF]], 0, 0 :: (dereferenceable invariant load (s256), align 8)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr_128 = COPY %7.sub0_sub1_sub2_sub3
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:sgpr_128 = COPY killed %7.sub4_sub5_sub6_sub7
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:sgpr_64 = COPY [[COPY]].sub0_sub1
+ ; GFX10-NEXT: [[COPY3:%[0-9]+]]:sgpr_64 = COPY killed [[COPY]].sub2_sub3
+ ; GFX10-NEXT: [[COPY4:%[0-9]+]]:sgpr_64 = COPY [[COPY1]].sub0_sub1
+ ; GFX10-NEXT: [[COPY5:%[0-9]+]]:sgpr_64 = COPY killed [[COPY1]].sub2_sub3
+ ;
+ ; GFX11-LABEL: name: merge_s_load_x2_x2_x2_x2
+ ; GFX11: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX11-NEXT: [[S_LOAD_DWORDX8_IMM:%[0-9]+]]:sgpr_256 = S_LOAD_DWORDX8_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s256), align 8)
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_128 = COPY [[S_LOAD_DWORDX8_IMM]].sub0_sub1_sub2_sub3
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_128 = COPY killed [[S_LOAD_DWORDX8_IMM]].sub4_sub5_sub6_sub7
+ ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sgpr_64 = COPY [[COPY]].sub0_sub1
+ ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sgpr_64 = COPY killed [[COPY]].sub2_sub3
+ ; GFX11-NEXT: [[COPY4:%[0-9]+]]:sgpr_64 = COPY [[COPY1]].sub0_sub1
+ ; GFX11-NEXT: [[COPY5:%[0-9]+]]:sgpr_64 = COPY killed [[COPY1]].sub2_sub3
+ ;
+ ; GFX12-LABEL: name: merge_s_load_x2_x2_x2_x2
+ ; GFX12: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[S_LOAD_DWORDX8_IMM:%[0-9]+]]:sgpr_256 = S_LOAD_DWORDX8_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s256), align 8)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sgpr_128 = COPY [[S_LOAD_DWORDX8_IMM]].sub0_sub1_sub2_sub3
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_128 = COPY killed [[S_LOAD_DWORDX8_IMM]].sub4_sub5_sub6_sub7
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_64 = COPY [[COPY]].sub0_sub1
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_64 = COPY killed [[COPY]].sub2_sub3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sgpr_64 = COPY [[COPY1]].sub0_sub1
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sgpr_64 = COPY killed [[COPY1]].sub2_sub3
%0:sgpr_64 = IMPLICIT_DEF
%1:sgpr_64 = S_LOAD_DWORDX2_IMM %0:sgpr_64, 0, 0 :: (dereferenceable invariant load (s64))
%2:sgpr_64 = S_LOAD_DWORDX2_IMM %0:sgpr_64, 8, 0 :: (dereferenceable invariant load (s64))
@@ -217,12 +302,181 @@ body: |
name: merge_s_load_x4_x4
body: |
bb.0:
- ; CHECK-LABEL: name: merge_s_load_x4_x4
- ; CHECK: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
- ; CHECK-NEXT: [[S_LOAD_DWORDX8_IMM:%[0-9]+]]:sgpr_256 = S_LOAD_DWORDX8_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s256), align 16)
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_128 = COPY [[S_LOAD_DWORDX8_IMM]].sub0_sub1_sub2_sub3
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr_128 = COPY killed [[S_LOAD_DWORDX8_IMM]].sub4_sub5_sub6_sub7
+ ; GFX10-LABEL: name: merge_s_load_x4_x4
+ ; GFX10: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX10-NEXT: early-clobber %3:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[DEF]], 0, 0 :: (dereferenceable invariant load (s256), align 16)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr_128 = COPY %3.sub0_sub1_sub2_sub3
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:sgpr_128 = COPY killed %3.sub4_sub5_sub6_sub7
+ ;
+ ; GFX11-LABEL: name: merge_s_load_x4_x4
+ ; GFX11: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX11-NEXT: [[S_LOAD_DWORDX8_IMM:%[0-9]+]]:sgpr_256 = S_LOAD_DWORDX8_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s256), align 16)
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_128 = COPY [[S_LOAD_DWORDX8_IMM]].sub0_sub1_sub2_sub3
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_128 = COPY killed [[S_LOAD_DWORDX8_IMM]].sub4_sub5_sub6_sub7
+ ;
+ ; GFX12-LABEL: name: merge_s_load_x4_x4
+ ; GFX12: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[S_LOAD_DWORDX8_IMM:%[0-9]+]]:sgpr_256 = S_LOAD_DWORDX8_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s256), align 16)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sgpr_128 = COPY [[S_LOAD_DWORDX8_IMM]].sub0_sub1_sub2_sub3
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_128 = COPY killed [[S_LOAD_DWORDX8_IMM]].sub4_sub5_sub6_sub7
%0:sgpr_64 = IMPLICIT_DEF
%1:sgpr_128 = S_LOAD_DWORDX4_IMM %0:sgpr_64, 0, 0 :: (dereferenceable invariant load (s128))
%2:sgpr_128 = S_LOAD_DWORDX4_IMM %0:sgpr_64, 16, 0 :: (dereferenceable invariant load (s128))
...
+
+# The constrained multi-dword scalar load merge tests.
+---
+name: merge_s_load_x1_x2ec
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: merge_s_load_x1_x2ec
+ ; CHECK: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s32))
+ ; CHECK-NEXT: early-clobber %2:sgpr_64 = S_LOAD_DWORDX2_IMM_ec [[DEF]], 4, 0 :: (dereferenceable invariant load (s64))
+ %0:sgpr_64 = IMPLICIT_DEF
+ %1:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0:sgpr_64, 0, 0 :: (dereferenceable invariant load (s32))
+ early-clobber %2:sgpr_64 = S_LOAD_DWORDX2_IMM_ec %0:sgpr_64, 4, 0 :: (dereferenceable invariant load (s64))
+...
+
+---
+name: merge_s_load_x1_x3ec
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: merge_s_load_x1_x3ec
+ ; CHECK: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s32))
+ ; CHECK-NEXT: early-clobber %2:sgpr_96 = S_LOAD_DWORDX3_IMM_ec [[DEF]], 4, 0 :: (dereferenceable invariant load (s96), align 16)
+ %0:sgpr_64 = IMPLICIT_DEF
+ %1:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0:sgpr_64, 0, 0 :: (dereferenceable invariant load (s32))
+ early-clobber %2:sgpr_96 = S_LOAD_DWORDX3_IMM_ec %0:sgpr_64, 4, 0 :: (dereferenceable invariant load (s96))
+...
+
+---
+name: merge_s_load_x2ec_x1
+body: |
+ bb.0:
+ ; GFX10-LABEL: name: merge_s_load_x2ec_x1
+ ; GFX10: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX10-NEXT: early-clobber %1:sgpr_64 = S_LOAD_DWORDX2_IMM_ec [[DEF]], 0, 0 :: (dereferenceable invariant load (s64))
+ ; GFX10-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[DEF]], 8, 0 :: (dereferenceable invariant load (s32))
+ ;
+ ; GFX11-LABEL: name: merge_s_load_x2ec_x1
+ ; GFX11: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX11-NEXT: early-clobber %1:sgpr_64 = S_LOAD_DWORDX2_IMM_ec [[DEF]], 0, 0 :: (dereferenceable invariant load (s64))
+ ; GFX11-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[DEF]], 8, 0 :: (dereferenceable invariant load (s32))
+ ;
+ ; GFX12-LABEL: name: merge_s_load_x2ec_x1
+ ; GFX12: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[S_LOAD_DWORDX3_IMM:%[0-9]+]]:sgpr_96 = S_LOAD_DWORDX3_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s96), align 8)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sgpr_64 = COPY [[S_LOAD_DWORDX3_IMM]].sub0_sub1
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32_xm0_xexec = COPY killed [[S_LOAD_DWORDX3_IMM]].sub2
+ %0:sgpr_64 = IMPLICIT_DEF
+ early-clobber %1:sgpr_64 = S_LOAD_DWORDX2_IMM_ec %0:sgpr_64, 0, 0 :: (dereferenceable invariant load (s64))
+ %2:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0:sgpr_64, 8, 0 :: (dereferenceable invariant load (s32))
+...
+
+---
+name: merge_s_load_x2ec_x2ec
+body: |
+ bb.0:
+ ; GFX10-LABEL: name: merge_s_load_x2ec_x2ec
+ ; GFX10: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX10-NEXT: early-clobber %3:sgpr_128 = S_LOAD_DWORDX4_IMM_ec [[DEF]], 0, 0 :: (dereferenceable invariant load (s128), align 8)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr_64 = COPY %3.sub0_sub1
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:sgpr_64 = COPY killed %3.sub2_sub3
+ ;
+ ; GFX11-LABEL: name: merge_s_load_x2ec_x2ec
+ ; GFX11: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX11-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s128), align 8)
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_64 = COPY [[S_LOAD_DWORDX4_IMM]].sub0_sub1
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_64 = COPY killed [[S_LOAD_DWORDX4_IMM]].sub2_sub3
+ ;
+ ; GFX12-LABEL: name: merge_s_load_x2ec_x2ec
+ ; GFX12: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s128), align 8)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sgpr_64 = COPY [[S_LOAD_DWORDX4_IMM]].sub0_sub1
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_64 = COPY killed [[S_LOAD_DWORDX4_IMM]].sub2_sub3
+ %0:sgpr_64 = IMPLICIT_DEF
+ early-clobber %1:sgpr_64 = S_LOAD_DWORDX2_IMM_ec %0:sgpr_64, 0, 0 :: (dereferenceable invariant load (s64))
+ early-clobber %2:sgpr_64 = S_LOAD_DWORDX2_IMM_ec %0:sgpr_64, 8, 0 :: (dereferenceable invariant load (s64))
+...
+
+---
+name: merge_s_load_x2ec_x2ec_x2ec_x2ec
+body: |
+ bb.0:
+ ; GFX10-LABEL: name: merge_s_load_x2ec_x2ec_x2ec_x2ec
+ ; GFX10: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX10-NEXT: early-clobber %7:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[DEF]], 0, 0 :: (dereferenceable invariant load (s256), align 8)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr_128 = COPY %7.sub0_sub1_sub2_sub3
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:sgpr_128 = COPY killed %7.sub4_sub5_sub6_sub7
+ ; GFX10-NEXT: [[COPY2:%[0-9]+]]:sgpr_64 = COPY [[COPY]].sub0_sub1
+ ; GFX10-NEXT: [[COPY3:%[0-9]+]]:sgpr_64 = COPY killed [[COPY]].sub2_sub3
+ ; GFX10-NEXT: [[COPY4:%[0-9]+]]:sgpr_64 = COPY [[COPY1]].sub0_sub1
+ ; GFX10-NEXT: [[COPY5:%[0-9]+]]:sgpr_64 = COPY killed [[COPY1]].sub2_sub3
+ ;
+ ; GFX11-LABEL: name: merge_s_load_x2ec_x2ec_x2ec_x2ec
+ ; GFX11: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX11-NEXT: [[S_LOAD_DWORDX8_IMM:%[0-9]+]]:sgpr_256 = S_LOAD_DWORDX8_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s256), align 8)
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_128 = COPY [[S_LOAD_DWORDX8_IMM]].sub0_sub1_sub2_sub3
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_128 = COPY killed [[S_LOAD_DWORDX8_IMM]].sub4_sub5_sub6_sub7
+ ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sgpr_64 = COPY [[COPY]].sub0_sub1
+ ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sgpr_64 = COPY killed [[COPY]].sub2_sub3
+ ; GFX11-NEXT: [[COPY4:%[0-9]+]]:sgpr_64 = COPY [[COPY1]].sub0_sub1
+ ; GFX11-NEXT: [[COPY5:%[0-9]+]]:sgpr_64 = COPY killed [[COPY1]].sub2_sub3
+ ;
+ ; GFX12-LABEL: name: merge_s_load_x2ec_x2ec_x2ec_x2ec
+ ; GFX12: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[S_LOAD_DWORDX8_IMM:%[0-9]+]]:sgpr_256 = S_LOAD_DWORDX8_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s256), align 8)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sgpr_128 = COPY [[S_LOAD_DWORDX8_IMM]].sub0_sub1_sub2_sub3
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_128 = COPY killed [[S_LOAD_DWORDX8_IMM]].sub4_sub5_sub6_sub7
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sgpr_64 = COPY [[COPY]].sub0_sub1
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sgpr_64 = COPY killed [[COPY]].sub2_sub3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sgpr_64 = COPY [[COPY1]].sub0_sub1
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sgpr_64 = COPY killed [[COPY1]].sub2_sub3
+ %0:sgpr_64 = IMPLICIT_DEF
+ early-clobber %1:sgpr_64 = S_LOAD_DWORDX2_IMM_ec %0:sgpr_64, 0, 0 :: (dereferenceable invariant load (s64))
+ early-clobber %2:sgpr_64 = S_LOAD_DWORDX2_IMM_ec %0:sgpr_64, 8, 0 :: (dereferenceable invariant load (s64))
+ early-clobber %3:sgpr_64 = S_LOAD_DWORDX2_IMM_ec %0:sgpr_64, 16, 0 :: (dereferenceable invariant load (s64))
+ early-clobber %4:sgpr_64 = S_LOAD_DWORDX2_IMM_ec %0:sgpr_64, 24, 0 :: (dereferenceable invariant load (s64))
+...
+
+---
+name: merge_s_load_x3ec_x1
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: merge_s_load_x3ec_x1
+ ; CHECK: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s128))
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_96 = COPY [[S_LOAD_DWORDX4_IMM]].sub0_sub1_sub2
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sreg_32_xm0_xexec = COPY killed [[S_LOAD_DWORDX4_IMM]].sub3
+ %0:sgpr_64 = IMPLICIT_DEF
+ early-clobber %1:sgpr_96 = S_LOAD_DWORDX3_IMM_ec %0:sgpr_64, 0, 0 :: (dereferenceable invariant load (s96))
+ %2:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0:sgpr_64, 12, 0 :: (dereferenceable invariant load (s32))
+...
+
+---
+name: merge_s_load_x4ec_x4ec
+body: |
+ bb.0:
+ ; GFX10-LABEL: name: merge_s_load_x4ec_x4ec
+ ; GFX10: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX10-NEXT: early-clobber %3:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[DEF]], 0, 0 :: (dereferenceable invariant load (s256), align 16)
+ ; GFX10-NEXT: [[COPY:%[0-9]+]]:sgpr_128 = COPY %3.sub0_sub1_sub2_sub3
+ ; GFX10-NEXT: [[COPY1:%[0-9]+]]:sgpr_128 = COPY killed %3.sub4_sub5_sub6_sub7
+ ;
+ ; GFX11-LABEL: name: merge_s_load_x4ec_x4ec
+ ; GFX11: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX11-NEXT: [[S_LOAD_DWORDX8_IMM:%[0-9]+]]:sgpr_256 = S_LOAD_DWORDX8_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s256), align 16)
+ ; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_128 = COPY [[S_LOAD_DWORDX8_IMM]].sub0_sub1_sub2_sub3
+ ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_128 = COPY killed [[S_LOAD_DWORDX8_IMM]].sub4_sub5_sub6_sub7
+ ;
+ ; GFX12-LABEL: name: merge_s_load_x4ec_x4ec
+ ; GFX12: [[DEF:%[0-9]+]]:sgpr_64 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[S_LOAD_DWORDX8_IMM:%[0-9]+]]:sgpr_256 = S_LOAD_DWORDX8_IMM [[DEF]], 0, 0 :: (dereferenceable invariant load (s256), align 16)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:sgpr_128 = COPY [[S_LOAD_DWORDX8_IMM]].sub0_sub1_sub2_sub3
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sgpr_128 = COPY killed [[S_LOAD_DWORDX8_IMM]].sub4_sub5_sub6_sub7
+ %0:sgpr_64 = IMPLICIT_DEF
+ early-clobber %1:sgpr_128 = S_LOAD_DWORDX4_IMM_ec %0:sgpr_64, 0, 0 :: (dereferenceable invariant load (s128))
+ early-clobber %2:sgpr_128 = S_LOAD_DWORDX4_IMM_ec %0:sgpr_64, 16, 0 :: (dereferenceable invariant load (s128))
+...
diff --git a/llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll b/llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll
index ece7e28c763fb..b696f097d05b7 100644
--- a/llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll
+++ b/llvm/test/CodeGen/AMDGPU/move-to-valu-ctlz-cttz.ll
@@ -8,17 +8,17 @@ declare i64 @llvm.cttz.i64(i64, i1) nounwind readnone
define amdgpu_kernel void @ctlz_i64_poison(ptr addrspace(1) noalias %out, ptr addrspace(1) nocapture readonly %arrayidx) nounwind {
; GFX9-LABEL: ctlz_i64_poison:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
-; GFX9-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
-; GFX9-NEXT: global_load_ubyte v3, v1, s[2:3] offset:7
-; GFX9-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
-; GFX9-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
-; GFX9-NEXT: global_load_ubyte v6, v1, s[2:3] offset:4
-; GFX9-NEXT: global_load_ubyte v7, v1, s[2:3]
-; GFX9-NEXT: global_load_ubyte v8, v1, s[2:3] offset:2
+; GFX9-NEXT: global_load_ubyte v0, v1, s[6:7] offset:5
+; GFX9-NEXT: global_load_ubyte v2, v1, s[6:7] offset:6
+; GFX9-NEXT: global_load_ubyte v3, v1, s[6:7] offset:7
+; GFX9-NEXT: global_load_ubyte v4, v1, s[6:7] offset:1
+; GFX9-NEXT: global_load_ubyte v5, v1, s[6:7] offset:3
+; GFX9-NEXT: global_load_ubyte v6, v1, s[6:7] offset:4
+; GFX9-NEXT: global_load_ubyte v7, v1, s[6:7]
+; GFX9-NEXT: global_load_ubyte v8, v1, s[6:7] offset:2
; GFX9-NEXT: s_waitcnt vmcnt(7)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 8, v0
; GFX9-NEXT: s_waitcnt vmcnt(5)
@@ -40,23 +40,23 @@ define amdgpu_kernel void @ctlz_i64_poison(ptr addrspace(1) noalias %out, ptr ad
; GFX9-NEXT: v_ffbh_u32_e32 v0, v0
; GFX9-NEXT: v_add_u32_e64 v2, v2, 32 clamp
; GFX9-NEXT: v_min_u32_e32 v0, v2, v0
-; GFX9-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX9-NEXT: global_store_dwordx2 v1, v[0:1], s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: ctlz_i64_poison:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_clause 0x7
-; GFX10-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
-; GFX10-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
-; GFX10-NEXT: global_load_ubyte v3, v1, s[2:3] offset:7
-; GFX10-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
-; GFX10-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
-; GFX10-NEXT: global_load_ubyte v6, v1, s[2:3]
-; GFX10-NEXT: global_load_ubyte v7, v1, s[2:3] offset:2
-; GFX10-NEXT: global_load_ubyte v8, v1, s[2:3] offset:4
+; GFX10-NEXT: global_load_ubyte v0, v1, s[6:7] offset:5
+; GFX10-NEXT: global_load_ubyte v2, v1, s[6:7] offset:6
+; GFX10-NEXT: global_load_ubyte v3, v1, s[6:7] offset:7
+; GFX10-NEXT: global_load_ubyte v4, v1, s[6:7] offset:1
+; GFX10-NEXT: global_load_ubyte v5, v1, s[6:7] offset:3
+; GFX10-NEXT: global_load_ubyte v6, v1, s[6:7]
+; GFX10-NEXT: global_load_ubyte v7, v1, s[6:7] offset:2
+; GFX10-NEXT: global_load_ubyte v8, v1, s[6:7] offset:4
; GFX10-NEXT: s_waitcnt vmcnt(7)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 8, v0
; GFX10-NEXT: s_waitcnt vmcnt(5)
@@ -76,7 +76,7 @@ define amdgpu_kernel void @ctlz_i64_poison(ptr addrspace(1) noalias %out, ptr ad
; GFX10-NEXT: v_ffbh_u32_e32 v0, v0
; GFX10-NEXT: v_add_nc_u32_e64 v2, v2, 32 clamp
; GFX10-NEXT: v_min_u32_e32 v0, v2, v0
-; GFX10-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX10-NEXT: global_store_dwordx2 v1, v[0:1], s[4:5]
; GFX10-NEXT: s_endpgm
%val = load i64, ptr addrspace(1) %arrayidx, align 1
%ctlz = tail call i64 @llvm.ctlz.i64(i64 %val, i1 true) nounwind readnone
@@ -87,17 +87,17 @@ define amdgpu_kernel void @ctlz_i64_poison(ptr addrspace(1) noalias %out, ptr ad
define amdgpu_kernel void @ctlz_i64(ptr addrspace(1) noalias %out, ptr addrspace(1) nocapture readonly %arrayidx) nounwind {
; GFX9-LABEL: ctlz_i64:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
-; GFX9-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
-; GFX9-NEXT: global_load_ubyte v3, v1, s[2:3] offset:7
-; GFX9-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
-; GFX9-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
-; GFX9-NEXT: global_load_ubyte v6, v1, s[2:3] offset:4
-; GFX9-NEXT: global_load_ubyte v7, v1, s[2:3]
-; GFX9-NEXT: global_load_ubyte v8, v1, s[2:3] offset:2
+; GFX9-NEXT: global_load_ubyte v0, v1, s[6:7] offset:5
+; GFX9-NEXT: global_load_ubyte v2, v1, s[6:7] offset:6
+; GFX9-NEXT: global_load_ubyte v3, v1, s[6:7] offset:7
+; GFX9-NEXT: global_load_ubyte v4, v1, s[6:7] offset:1
+; GFX9-NEXT: global_load_ubyte v5, v1, s[6:7] offset:3
+; GFX9-NEXT: global_load_ubyte v6, v1, s[6:7] offset:4
+; GFX9-NEXT: global_load_ubyte v7, v1, s[6:7]
+; GFX9-NEXT: global_load_ubyte v8, v1, s[6:7] offset:2
; GFX9-NEXT: s_waitcnt vmcnt(7)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 8, v0
; GFX9-NEXT: s_waitcnt vmcnt(5)
@@ -120,23 +120,23 @@ define amdgpu_kernel void @ctlz_i64(ptr addrspace(1) noalias %out, ptr addrspace
; GFX9-NEXT: v_add_u32_e64 v2, v2, 32 clamp
; GFX9-NEXT: v_min_u32_e32 v0, v2, v0
; GFX9-NEXT: v_min_u32_e32 v0, 64, v0
-; GFX9-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX9-NEXT: global_store_dwordx2 v1, v[0:1], s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: ctlz_i64:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_clause 0x7
-; GFX10-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
-; GFX10-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
-; GFX10-NEXT: global_load_ubyte v3, v1, s[2:3] offset:7
-; GFX10-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
-; GFX10-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
-; GFX10-NEXT: global_load_ubyte v6, v1, s[2:3]
-; GFX10-NEXT: global_load_ubyte v7, v1, s[2:3] offset:2
-; GFX10-NEXT: global_load_ubyte v8, v1, s[2:3] offset:4
+; GFX10-NEXT: global_load_ubyte v0, v1, s[6:7] offset:5
+; GFX10-NEXT: global_load_ubyte v2, v1, s[6:7] offset:6
+; GFX10-NEXT: global_load_ubyte v3, v1, s[6:7] offset:7
+; GFX10-NEXT: global_load_ubyte v4, v1, s[6:7] offset:1
+; GFX10-NEXT: global_load_ubyte v5, v1, s[6:7] offset:3
+; GFX10-NEXT: global_load_ubyte v6, v1, s[6:7]
+; GFX10-NEXT: global_load_ubyte v7, v1, s[6:7] offset:2
+; GFX10-NEXT: global_load_ubyte v8, v1, s[6:7] offset:4
; GFX10-NEXT: s_waitcnt vmcnt(7)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 8, v0
; GFX10-NEXT: s_waitcnt vmcnt(5)
@@ -157,7 +157,7 @@ define amdgpu_kernel void @ctlz_i64(ptr addrspace(1) noalias %out, ptr addrspace
; GFX10-NEXT: v_add_nc_u32_e64 v2, v2, 32 clamp
; GFX10-NEXT: v_min_u32_e32 v0, v2, v0
; GFX10-NEXT: v_min_u32_e32 v0, 64, v0
-; GFX10-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX10-NEXT: global_store_dwordx2 v1, v[0:1], s[4:5]
; GFX10-NEXT: s_endpgm
%val = load i64, ptr addrspace(1) %arrayidx, align 1
%ctlz = tail call i64 @llvm.ctlz.i64(i64 %val, i1 false) nounwind readnone
@@ -168,17 +168,17 @@ define amdgpu_kernel void @ctlz_i64(ptr addrspace(1) noalias %out, ptr addrspace
define amdgpu_kernel void @cttz_i64_poison(ptr addrspace(1) noalias %out, ptr addrspace(1) nocapture readonly %arrayidx) nounwind {
; GFX9-LABEL: cttz_i64_poison:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
-; GFX9-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
-; GFX9-NEXT: global_load_ubyte v3, v1, s[2:3] offset:7
-; GFX9-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
-; GFX9-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
-; GFX9-NEXT: global_load_ubyte v6, v1, s[2:3] offset:4
-; GFX9-NEXT: global_load_ubyte v7, v1, s[2:3]
-; GFX9-NEXT: global_load_ubyte v8, v1, s[2:3] offset:2
+; GFX9-NEXT: global_load_ubyte v0, v1, s[6:7] offset:5
+; GFX9-NEXT: global_load_ubyte v2, v1, s[6:7] offset:6
+; GFX9-NEXT: global_load_ubyte v3, v1, s[6:7] offset:7
+; GFX9-NEXT: global_load_ubyte v4, v1, s[6:7] offset:1
+; GFX9-NEXT: global_load_ubyte v5, v1, s[6:7] offset:3
+; GFX9-NEXT: global_load_ubyte v6, v1, s[6:7] offset:4
+; GFX9-NEXT: global_load_ubyte v7, v1, s[6:7]
+; GFX9-NEXT: global_load_ubyte v8, v1, s[6:7] offset:2
; GFX9-NEXT: s_waitcnt vmcnt(7)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 8, v0
; GFX9-NEXT: s_waitcnt vmcnt(5)
@@ -200,23 +200,23 @@ define amdgpu_kernel void @cttz_i64_poison(ptr addrspace(1) noalias %out, ptr ad
; GFX9-NEXT: v_ffbl_b32_e32 v2, v2
; GFX9-NEXT: v_add_u32_e64 v0, v0, 32 clamp
; GFX9-NEXT: v_min_u32_e32 v0, v0, v2
-; GFX9-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX9-NEXT: global_store_dwordx2 v1, v[0:1], s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: cttz_i64_poison:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_clause 0x7
-; GFX10-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
-; GFX10-NEXT: global_load_ubyte v2, v1, s[2:3] offset:7
-; GFX10-NEXT: global_load_ubyte v3, v1, s[2:3] offset:6
-; GFX10-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
-; GFX10-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
-; GFX10-NEXT: global_load_ubyte v6, v1, s[2:3] offset:4
-; GFX10-NEXT: global_load_ubyte v7, v1, s[2:3]
-; GFX10-NEXT: global_load_ubyte v8, v1, s[2:3] offset:2
+; GFX10-NEXT: global_load_ubyte v0, v1, s[6:7] offset:5
+; GFX10-NEXT: global_load_ubyte v2, v1, s[6:7] offset:7
+; GFX10-NEXT: global_load_ubyte v3, v1, s[6:7] offset:6
+; GFX10-NEXT: global_load_ubyte v4, v1, s[6:7] offset:1
+; GFX10-NEXT: global_load_ubyte v5, v1, s[6:7] offset:3
+; GFX10-NEXT: global_load_ubyte v6, v1, s[6:7] offset:4
+; GFX10-NEXT: global_load_ubyte v7, v1, s[6:7]
+; GFX10-NEXT: global_load_ubyte v8, v1, s[6:7] offset:2
; GFX10-NEXT: s_waitcnt vmcnt(7)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 8, v0
; GFX10-NEXT: s_waitcnt vmcnt(6)
@@ -238,7 +238,7 @@ define amdgpu_kernel void @cttz_i64_poison(ptr addrspace(1) noalias %out, ptr ad
; GFX10-NEXT: v_ffbl_b32_e32 v2, v2
; GFX10-NEXT: v_add_nc_u32_e64 v0, v0, 32 clamp
; GFX10-NEXT: v_min_u32_e32 v0, v0, v2
-; GFX10-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX10-NEXT: global_store_dwordx2 v1, v[0:1], s[4:5]
; GFX10-NEXT: s_endpgm
%val = load i64, ptr addrspace(1) %arrayidx, align 1
%cttz = tail call i64 @llvm.cttz.i64(i64 %val, i1 true) nounwind readnone
@@ -249,17 +249,17 @@ define amdgpu_kernel void @cttz_i64_poison(ptr addrspace(1) noalias %out, ptr ad
define amdgpu_kernel void @cttz_i64(ptr addrspace(1) noalias %out, ptr addrspace(1) nocapture readonly %arrayidx) nounwind {
; GFX9-LABEL: cttz_i64:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
-; GFX9-NEXT: global_load_ubyte v2, v1, s[2:3] offset:6
-; GFX9-NEXT: global_load_ubyte v3, v1, s[2:3] offset:7
-; GFX9-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
-; GFX9-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
-; GFX9-NEXT: global_load_ubyte v6, v1, s[2:3] offset:4
-; GFX9-NEXT: global_load_ubyte v7, v1, s[2:3]
-; GFX9-NEXT: global_load_ubyte v8, v1, s[2:3] offset:2
+; GFX9-NEXT: global_load_ubyte v0, v1, s[6:7] offset:5
+; GFX9-NEXT: global_load_ubyte v2, v1, s[6:7] offset:6
+; GFX9-NEXT: global_load_ubyte v3, v1, s[6:7] offset:7
+; GFX9-NEXT: global_load_ubyte v4, v1, s[6:7] offset:1
+; GFX9-NEXT: global_load_ubyte v5, v1, s[6:7] offset:3
+; GFX9-NEXT: global_load_ubyte v6, v1, s[6:7] offset:4
+; GFX9-NEXT: global_load_ubyte v7, v1, s[6:7]
+; GFX9-NEXT: global_load_ubyte v8, v1, s[6:7] offset:2
; GFX9-NEXT: s_waitcnt vmcnt(7)
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 8, v0
; GFX9-NEXT: s_waitcnt vmcnt(5)
@@ -282,23 +282,23 @@ define amdgpu_kernel void @cttz_i64(ptr addrspace(1) noalias %out, ptr addrspace
; GFX9-NEXT: v_add_u32_e64 v0, v0, 32 clamp
; GFX9-NEXT: v_min_u32_e32 v0, v0, v2
; GFX9-NEXT: v_min_u32_e32 v0, 64, v0
-; GFX9-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX9-NEXT: global_store_dwordx2 v1, v[0:1], s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: cttz_i64:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_clause 0x7
-; GFX10-NEXT: global_load_ubyte v0, v1, s[2:3] offset:5
-; GFX10-NEXT: global_load_ubyte v2, v1, s[2:3] offset:7
-; GFX10-NEXT: global_load_ubyte v3, v1, s[2:3] offset:6
-; GFX10-NEXT: global_load_ubyte v4, v1, s[2:3] offset:1
-; GFX10-NEXT: global_load_ubyte v5, v1, s[2:3] offset:3
-; GFX10-NEXT: global_load_ubyte v6, v1, s[2:3] offset:4
-; GFX10-NEXT: global_load_ubyte v7, v1, s[2:3]
-; GFX10-NEXT: global_load_ubyte v8, v1, s[2:3] offset:2
+; GFX10-NEXT: global_load_ubyte v0, v1, s[6:7] offset:5
+; GFX10-NEXT: global_load_ubyte v2, v1, s[6:7] offset:7
+; GFX10-NEXT: global_load_ubyte v3, v1, s[6:7] offset:6
+; GFX10-NEXT: global_load_ubyte v4, v1, s[6:7] offset:1
+; GFX10-NEXT: global_load_ubyte v5, v1, s[6:7] offset:3
+; GFX10-NEXT: global_load_ubyte v6, v1, s[6:7] offset:4
+; GFX10-NEXT: global_load_ubyte v7, v1, s[6:7]
+; GFX10-NEXT: global_load_ubyte v8, v1, s[6:7] offset:2
; GFX10-NEXT: s_waitcnt vmcnt(7)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 8, v0
; GFX10-NEXT: s_waitcnt vmcnt(6)
@@ -321,7 +321,7 @@ define amdgpu_kernel void @cttz_i64(ptr addrspace(1) noalias %out, ptr addrspace
; GFX10-NEXT: v_add_nc_u32_e64 v0, v0, 32 clamp
; GFX10-NEXT: v_min_u32_e32 v0, v0, v2
; GFX10-NEXT: v_min_u32_e32 v0, 64, v0
-; GFX10-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX10-NEXT: global_store_dwordx2 v1, v[0:1], s[4:5]
; GFX10-NEXT: s_endpgm
%val = load i64, ptr addrspace(1) %arrayidx, align 1
%cttz = tail call i64 @llvm.cttz.i64(i64 %val, i1 false) nounwind readnone
diff --git a/llvm/test/CodeGen/AMDGPU/mul_int24.ll b/llvm/test/CodeGen/AMDGPU/mul_int24.ll
index 842dc36e00154..be77a10380c49 100644
--- a/llvm/test/CodeGen/AMDGPU/mul_int24.ll
+++ b/llvm/test/CodeGen/AMDGPU/mul_int24.ll
@@ -39,17 +39,17 @@ define amdgpu_kernel void @test_smul24_i32(ptr addrspace(1) %out, i32 %a, i32 %b
;
; GFX9-LABEL: test_smul24_i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX9-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX9-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-NEXT: s_mov_b32 s2, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s4, s0
-; GFX9-NEXT: s_mov_b32 s5, s1
-; GFX9-NEXT: s_bfe_i32 s0, s2, 0x180000
-; GFX9-NEXT: s_bfe_i32 s1, s3, 0x180000
-; GFX9-NEXT: s_mul_i32 s0, s0, s1
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX9-NEXT: s_mov_b32 s0, s4
+; GFX9-NEXT: s_mov_b32 s1, s5
+; GFX9-NEXT: s_bfe_i32 s4, s6, 0x180000
+; GFX9-NEXT: s_bfe_i32 s5, s7, 0x180000
+; GFX9-NEXT: s_mul_i32 s4, s4, s5
+; GFX9-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
; EG-LABEL: test_smul24_i32:
@@ -126,17 +126,17 @@ define amdgpu_kernel void @test_smulhi24_i64(ptr addrspace(1) %out, i32 %a, i32
;
; GFX9-LABEL: test_smulhi24_i64:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX9-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX9-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-NEXT: s_mov_b32 s2, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s4, s0
-; GFX9-NEXT: s_mov_b32 s5, s1
-; GFX9-NEXT: s_bfe_i32 s0, s2, 0x180000
-; GFX9-NEXT: s_bfe_i32 s1, s3, 0x180000
-; GFX9-NEXT: s_mul_hi_i32 s0, s0, s1
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX9-NEXT: s_mov_b32 s0, s4
+; GFX9-NEXT: s_mov_b32 s1, s5
+; GFX9-NEXT: s_bfe_i32 s4, s6, 0x180000
+; GFX9-NEXT: s_bfe_i32 s5, s7, 0x180000
+; GFX9-NEXT: s_mul_hi_i32 s4, s4, s5
+; GFX9-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
; EG-LABEL: test_smulhi24_i64:
diff --git a/llvm/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll b/llvm/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll
index 0c0bb830ba847..342f36b6fa622 100644
--- a/llvm/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll
+++ b/llvm/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll
@@ -39,17 +39,17 @@ define amdgpu_kernel void @test_umul24_i32(ptr addrspace(1) %out, i32 %a, i32 %b
;
; GFX9-LABEL: test_umul24_i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX9-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX9-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-NEXT: s_mov_b32 s2, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s4, s0
-; GFX9-NEXT: s_mov_b32 s5, s1
-; GFX9-NEXT: s_and_b32 s0, s2, 0xffffff
-; GFX9-NEXT: s_and_b32 s1, s3, 0xffffff
-; GFX9-NEXT: s_mul_i32 s0, s0, s1
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX9-NEXT: s_mov_b32 s0, s4
+; GFX9-NEXT: s_mov_b32 s1, s5
+; GFX9-NEXT: s_and_b32 s4, s6, 0xffffff
+; GFX9-NEXT: s_and_b32 s5, s7, 0xffffff
+; GFX9-NEXT: s_mul_i32 s4, s4, s5
+; GFX9-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
entry:
%0 = shl i32 %a, 8
@@ -405,17 +405,17 @@ define amdgpu_kernel void @test_umulhi24_i32_i64(ptr addrspace(1) %out, i32 %a,
;
; GFX9-LABEL: test_umulhi24_i32_i64:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX9-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX9-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-NEXT: s_mov_b32 s2, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s4, s0
-; GFX9-NEXT: s_mov_b32 s5, s1
-; GFX9-NEXT: s_and_b32 s0, s2, 0xffffff
-; GFX9-NEXT: s_and_b32 s1, s3, 0xffffff
-; GFX9-NEXT: s_mul_hi_u32 s0, s0, s1
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX9-NEXT: s_mov_b32 s0, s4
+; GFX9-NEXT: s_mov_b32 s1, s5
+; GFX9-NEXT: s_and_b32 s4, s6, 0xffffff
+; GFX9-NEXT: s_and_b32 s5, s7, 0xffffff
+; GFX9-NEXT: s_mul_hi_u32 s4, s4, s5
+; GFX9-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
entry:
%a.24 = and i32 %a, 16777215
@@ -661,14 +661,14 @@ define amdgpu_kernel void @test_umulhi16_i32(ptr addrspace(1) %out, i32 %a, i32
;
; GFX9-LABEL: test_umulhi16_i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX9-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX9-NEXT: s_mul_i32 s2, s2, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: global_store_short_d16_hi v0, v1, s[0:1]
+; GFX9-NEXT: s_and_b32 s0, s6, 0xffff
+; GFX9-NEXT: s_and_b32 s1, s7, 0xffff
+; GFX9-NEXT: s_mul_i32 s0, s0, s1
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: global_store_short_d16_hi v0, v1, s[4:5]
; GFX9-NEXT: s_endpgm
entry:
%a.16 = and i32 %a, 65535
diff --git a/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll b/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
index c72a7ba3eee83..79082a54c6a36 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
@@ -4,17 +4,17 @@
define amdgpu_kernel void @fma_vector_vector_scalar_lo(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 {
; GCN-LABEL: fma_vector_vector_scalar_lo:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GCN-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s2
-; GCN-NEXT: v_mov_b32_e32 v1, s3
+; GCN-NEXT: v_mov_b32_e32 v0, s6
+; GCN-NEXT: v_mov_b32_e32 v1, s7
; GCN-NEXT: ds_read_b32 v2, v0
; GCN-NEXT: ds_read_b32 v0, v0 offset:4
; GCN-NEXT: ds_read_u16 v1, v1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_pk_fma_f16 v0, v2, v0, v1 op_sel_hi:[1,1,0]
-; GCN-NEXT: global_store_dword v3, v0, s[0:1]
+; GCN-NEXT: global_store_dword v3, v0, s[4:5]
; GCN-NEXT: s_endpgm
bb:
%lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
@@ -35,17 +35,17 @@ bb:
define amdgpu_kernel void @fma_vector_vector_neg_broadcast_scalar_lo(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 {
; GCN-LABEL: fma_vector_vector_neg_broadcast_scalar_lo:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GCN-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s2
-; GCN-NEXT: v_mov_b32_e32 v1, s3
+; GCN-NEXT: v_mov_b32_e32 v0, s6
+; GCN-NEXT: v_mov_b32_e32 v1, s7
; GCN-NEXT: ds_read_b32 v2, v0
; GCN-NEXT: ds_read_b32 v0, v0 offset:4
; GCN-NEXT: ds_read_u16 v1, v1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_pk_fma_f16 v0, v2, v0, v1 op_sel_hi:[1,1,0] neg_lo:[0,0,1] neg_hi:[0,0,1]
-; GCN-NEXT: global_store_dword v3, v0, s[0:1]
+; GCN-NEXT: global_store_dword v3, v0, s[4:5]
; GCN-NEXT: s_endpgm
bb:
%lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
@@ -67,17 +67,17 @@ bb:
define amdgpu_kernel void @fma_vector_vector_neg_scalar_lo(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 {
; GCN-LABEL: fma_vector_vector_neg_scalar_lo:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GCN-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s2
-; GCN-NEXT: v_mov_b32_e32 v1, s3
+; GCN-NEXT: v_mov_b32_e32 v0, s6
+; GCN-NEXT: v_mov_b32_e32 v1, s7
; GCN-NEXT: ds_read_b32 v2, v0
; GCN-NEXT: ds_read_b32 v0, v0 offset:4
; GCN-NEXT: ds_read_u16 v1, v1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_pk_fma_f16 v0, v2, v0, v1 op_sel_hi:[1,1,0] neg_lo:[0,0,1] neg_hi:[0,0,1]
-; GCN-NEXT: global_store_dword v3, v0, s[0:1]
+; GCN-NEXT: global_store_dword v3, v0, s[4:5]
; GCN-NEXT: s_endpgm
bb:
%lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
@@ -99,17 +99,17 @@ bb:
define amdgpu_kernel void @fma_vector_vector_neg_broadcast_neg_scalar_lo(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 {
; GCN-LABEL: fma_vector_vector_neg_broadcast_neg_scalar_lo:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GCN-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s2
-; GCN-NEXT: v_mov_b32_e32 v1, s3
+; GCN-NEXT: v_mov_b32_e32 v0, s6
+; GCN-NEXT: v_mov_b32_e32 v1, s7
; GCN-NEXT: ds_read_b32 v2, v0
; GCN-NEXT: ds_read_b32 v0, v0 offset:4
; GCN-NEXT: ds_read_u16 v1, v1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_pk_fma_f16 v0, v2, v0, v1 op_sel_hi:[1,1,0]
-; GCN-NEXT: global_store_dword v3, v0, s[0:1]
+; GCN-NEXT: global_store_dword v3, v0, s[4:5]
; GCN-NEXT: s_endpgm
bb:
%lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
@@ -132,17 +132,17 @@ bb:
define amdgpu_kernel void @fma_vector_vector_scalar_neg_lo(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 {
; GCN-LABEL: fma_vector_vector_scalar_neg_lo:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GCN-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s2
-; GCN-NEXT: v_mov_b32_e32 v1, s3
+; GCN-NEXT: v_mov_b32_e32 v0, s6
+; GCN-NEXT: v_mov_b32_e32 v1, s7
; GCN-NEXT: ds_read_b32 v2, v0
; GCN-NEXT: ds_read_b32 v0, v0 offset:4
; GCN-NEXT: ds_read_u16 v1, v1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_pk_fma_f16 v0, v2, v0, v1 op_sel_hi:[1,1,0] neg_lo:[0,0,1]
-; GCN-NEXT: global_store_dword v3, v0, s[0:1]
+; GCN-NEXT: global_store_dword v3, v0, s[4:5]
; GCN-NEXT: s_endpgm
bb:
%lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
@@ -163,17 +163,17 @@ bb:
define amdgpu_kernel void @fma_vector_vector_scalar_neg_hi(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 {
; GCN-LABEL: fma_vector_vector_scalar_neg_hi:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GCN-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s2
-; GCN-NEXT: v_mov_b32_e32 v1, s3
+; GCN-NEXT: v_mov_b32_e32 v0, s6
+; GCN-NEXT: v_mov_b32_e32 v1, s7
; GCN-NEXT: ds_read_b32 v2, v0
; GCN-NEXT: ds_read_b32 v0, v0 offset:4
; GCN-NEXT: ds_read_u16 v1, v1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_pk_fma_f16 v0, v2, v0, v1 op_sel_hi:[1,1,0] neg_hi:[0,0,1]
-; GCN-NEXT: global_store_dword v3, v0, s[0:1]
+; GCN-NEXT: global_store_dword v3, v0, s[4:5]
; GCN-NEXT: s_endpgm
bb:
%lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
@@ -194,16 +194,16 @@ bb:
define amdgpu_kernel void @add_vector_neg_bitcast_scalar_lo(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 {
; GCN-LABEL: add_vector_neg_bitcast_scalar_lo:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GCN-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s2
-; GCN-NEXT: v_mov_b32_e32 v1, s3
+; GCN-NEXT: v_mov_b32_e32 v0, s6
+; GCN-NEXT: v_mov_b32_e32 v1, s7
; GCN-NEXT: ds_read_b32 v0, v0
; GCN-NEXT: ds_read_u16 v1, v1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_pk_add_u16 v0, v0, v1 op_sel_hi:[1,0] neg_lo:[0,1] neg_hi:[0,1]
-; GCN-NEXT: global_store_dword v2, v0, s[0:1]
+; GCN-NEXT: global_store_dword v2, v0, s[4:5]
; GCN-NEXT: s_endpgm
bb:
%vec0 = load volatile <2 x i16>, ptr addrspace(3) %lds, align 4
@@ -222,11 +222,11 @@ bb:
define amdgpu_kernel void @fma_vector_vector_scalar_lo_neg_scalar_hi(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 {
; GCN-LABEL: fma_vector_vector_scalar_lo_neg_scalar_hi:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GCN-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GCN-NEXT: v_mov_b32_e32 v4, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s2
-; GCN-NEXT: v_mov_b32_e32 v1, s3
+; GCN-NEXT: v_mov_b32_e32 v0, s6
+; GCN-NEXT: v_mov_b32_e32 v1, s7
; GCN-NEXT: ds_read_b32 v2, v0
; GCN-NEXT: ds_read_b32 v0, v0 offset:4
; GCN-NEXT: ds_read_u16 v3, v1
@@ -237,7 +237,7 @@ define amdgpu_kernel void @fma_vector_vector_scalar_lo_neg_scalar_hi(ptr addrspa
; GCN-NEXT: v_xor_b32_e32 v1, 0x8000, v1
; GCN-NEXT: v_lshl_or_b32 v1, v1, 16, v3
; GCN-NEXT: v_pk_fma_f16 v0, v2, v0, v1
-; GCN-NEXT: global_store_dword v4, v0, s[0:1]
+; GCN-NEXT: global_store_dword v4, v0, s[4:5]
; GCN-NEXT: s_endpgm
bb:
%lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
@@ -261,10 +261,10 @@ bb:
define amdgpu_kernel void @fma_vector_vector_neg_scalar_lo_scalar_hi(ptr addrspace(1) %out, ptr addrspace(3) %lds, ptr addrspace(3) %arg2) #0 {
; GCN-LABEL: fma_vector_vector_neg_scalar_lo_scalar_hi:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GCN-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s2
-; GCN-NEXT: v_mov_b32_e32 v1, s3
+; GCN-NEXT: v_mov_b32_e32 v0, s6
+; GCN-NEXT: v_mov_b32_e32 v1, s7
; GCN-NEXT: ds_read_b32 v2, v0
; GCN-NEXT: ds_read_b32 v0, v0 offset:4
; GCN-NEXT: ds_read_u16 v3, v1
@@ -273,7 +273,7 @@ define amdgpu_kernel void @fma_vector_vector_neg_scalar_lo_scalar_hi(ptr addrspa
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_pk_fma_f16 v0, v2, v0, v3 neg_lo:[0,0,1] neg_hi:[0,0,1]
-; GCN-NEXT: global_store_dword v1, v0, s[0:1]
+; GCN-NEXT: global_store_dword v1, v0, s[4:5]
; GCN-NEXT: s_endpgm
bb:
%lds.gep1 = getelementptr inbounds <2 x half>, ptr addrspace(3) %lds, i32 1
diff --git a/llvm/test/CodeGen/AMDGPU/ptr-buffer-alias-scheduling.ll b/llvm/test/CodeGen/AMDGPU/ptr-buffer-alias-scheduling.ll
index 9a8d5acfbe3e9..011bb332ddd0a 100644
--- a/llvm/test/CodeGen/AMDGPU/ptr-buffer-alias-scheduling.ll
+++ b/llvm/test/CodeGen/AMDGPU/ptr-buffer-alias-scheduling.ll
@@ -5,28 +5,28 @@
define amdgpu_kernel void @buffers_dont_alias(ptr addrspace(8) noalias %a, ptr addrspace(8) noalias %b) {
; SDAG-LABEL: buffers_dont_alias:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
+; SDAG-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0
+; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[4:7], 0
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: v_mul_f32_e32 v0, v0, v0
; SDAG-NEXT: v_mul_f32_e32 v1, v1, v1
; SDAG-NEXT: v_mul_f32_e32 v2, v2, v2
; SDAG-NEXT: v_mul_f32_e32 v3, v3, v3
-; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: buffers_dont_alias:
; GISEL: ; %bb.0:
-; GISEL-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x24
+; GISEL-NEXT: s_load_dwordx8 s[4:11], s[2:3], 0x24
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0
+; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[4:7], 0
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: v_mul_f32_e32 v0, v0, v0
; GISEL-NEXT: v_mul_f32_e32 v1, v1, v1
; GISEL-NEXT: v_mul_f32_e32 v2, v2, v2
; GISEL-NEXT: v_mul_f32_e32 v3, v3, v3
-; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0
; GISEL-NEXT: s_endpgm
%l0 = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) %a, i32 0, i32 0, i32 0)
%s0 = fmul float %l0, %l0
@@ -50,40 +50,40 @@ define amdgpu_kernel void @buffers_dont_alias(ptr addrspace(8) noalias %a, ptr a
define amdgpu_kernel void @buffers_from_flat_dont_alias(ptr noalias %a.flat, ptr noalias %b.flat) {
; SDAG-LABEL: buffers_from_flat_dont_alias:
; SDAG: ; %bb.0:
-; SDAG-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; SDAG-NEXT: s_mov_b32 s7, 0
-; SDAG-NEXT: s_mov_b32 s6, 16
+; SDAG-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; SDAG-NEXT: s_mov_b32 s3, 0
+; SDAG-NEXT: s_mov_b32 s2, 16
; SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-NEXT: s_and_b32 s5, s1, 0xffff
-; SDAG-NEXT: s_mov_b32 s4, s0
-; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[4:7], 0
-; SDAG-NEXT: s_and_b32 s5, s3, 0xffff
-; SDAG-NEXT: s_mov_b32 s4, s2
+; SDAG-NEXT: s_and_b32 s1, s5, 0xffff
+; SDAG-NEXT: s_mov_b32 s0, s4
+; SDAG-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0
+; SDAG-NEXT: s_and_b32 s1, s7, 0xffff
+; SDAG-NEXT: s_mov_b32 s0, s6
; SDAG-NEXT: s_waitcnt vmcnt(0)
; SDAG-NEXT: v_mul_f32_e32 v0, v0, v0
; SDAG-NEXT: v_mul_f32_e32 v1, v1, v1
; SDAG-NEXT: v_mul_f32_e32 v2, v2, v2
; SDAG-NEXT: v_mul_f32_e32 v3, v3, v3
-; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; SDAG-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; SDAG-NEXT: s_endpgm
;
; GISEL-LABEL: buffers_from_flat_dont_alias:
; GISEL: ; %bb.0:
-; GISEL-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GISEL-NEXT: s_mov_b32 s7, 0
-; GISEL-NEXT: s_mov_b32 s6, 16
+; GISEL-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GISEL-NEXT: s_mov_b32 s3, 0
+; GISEL-NEXT: s_mov_b32 s2, 16
; GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-NEXT: s_and_b32 s5, s1, 0xffff
-; GISEL-NEXT: s_mov_b32 s4, s0
-; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[4:7], 0
-; GISEL-NEXT: s_and_b32 s5, s3, 0xffff
-; GISEL-NEXT: s_mov_b32 s4, s2
+; GISEL-NEXT: s_and_b32 s1, s5, 0xffff
+; GISEL-NEXT: s_mov_b32 s0, s4
+; GISEL-NEXT: buffer_load_dwordx4 v[0:3], off, s[0:3], 0
+; GISEL-NEXT: s_and_b32 s1, s7, 0xffff
+; GISEL-NEXT: s_mov_b32 s0, s6
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: v_mul_f32_e32 v0, v0, v0
; GISEL-NEXT: v_mul_f32_e32 v1, v1, v1
; GISEL-NEXT: v_mul_f32_e32 v2, v2, v2
; GISEL-NEXT: v_mul_f32_e32 v3, v3, v3
-; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GISEL-NEXT: s_endpgm
%a = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p0(ptr %a.flat, i16 0, i32 16, i32 0)
%b = call ptr addrspace(8) @llvm.amdgcn.make.buffer.rsrc.p0(ptr %b.flat, i16 0, i32 16, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/rotl.ll b/llvm/test/CodeGen/AMDGPU/rotl.ll
index fdce4431fbbf2..f692584ef92be 100644
--- a/llvm/test/CodeGen/AMDGPU/rotl.ll
+++ b/llvm/test/CodeGen/AMDGPU/rotl.ll
@@ -47,12 +47,12 @@ define amdgpu_kernel void @rotl_i32(ptr addrspace(1) %in, i32 %x, i32 %y) {
;
; GFX10-LABEL: rotl_i32:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_sub_i32 s3, 32, s3
-; GFX10-NEXT: v_alignbit_b32 v1, s2, s2, s3
-; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_sub_i32 s0, 32, s7
+; GFX10-NEXT: v_alignbit_b32 v1, s6, s6, s0
+; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: rotl_i32:
diff --git a/llvm/test/CodeGen/AMDGPU/rotr.ll b/llvm/test/CodeGen/AMDGPU/rotr.ll
index 0e1dd69d930ae..a368aa1055b28 100644
--- a/llvm/test/CodeGen/AMDGPU/rotr.ll
+++ b/llvm/test/CodeGen/AMDGPU/rotr.ll
@@ -43,11 +43,11 @@ define amdgpu_kernel void @rotr_i32(ptr addrspace(1) %in, i32 %x, i32 %y) {
;
; GFX10-LABEL: rotr_i32:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_alignbit_b32 v1, s2, s2, s3
-; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: v_alignbit_b32 v1, s6, s6, s7
+; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: rotr_i32:
diff --git a/llvm/test/CodeGen/AMDGPU/shl.v2i16.ll b/llvm/test/CodeGen/AMDGPU/shl.v2i16.ll
index 8c663d963b73e..4952f80c0b411 100644
--- a/llvm/test/CodeGen/AMDGPU/shl.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/shl.v2i16.ll
@@ -8,15 +8,15 @@
define amdgpu_kernel void @s_shl_v2i16(ptr addrspace(1) %out, <2 x i16> %lhs, <2 x i16> %rhs) #0 {
; GFX9-LABEL: s_shl_v2i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX9-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX9-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-NEXT: s_mov_b32 s2, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: s_mov_b32 s4, s0
-; GFX9-NEXT: s_mov_b32 s5, s1
-; GFX9-NEXT: v_pk_lshlrev_b16 v0, s3, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-NEXT: s_mov_b32 s0, s4
+; GFX9-NEXT: s_mov_b32 s1, s5
+; GFX9-NEXT: v_pk_lshlrev_b16 v0, s7, v0
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
; VI-LABEL: s_shl_v2i16:
@@ -59,14 +59,14 @@ define amdgpu_kernel void @s_shl_v2i16(ptr addrspace(1) %out, <2 x i16> %lhs, <2
;
; GFX10-LABEL: s_shl_v2i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-NEXT: s_mov_b32 s7, 0x31016000
-; GFX10-NEXT: s_mov_b32 s6, -1
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-NEXT: s_mov_b32 s3, 0x31016000
+; GFX10-NEXT: s_mov_b32 s2, -1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_pk_lshlrev_b16 v0, s3, s2
-; GFX10-NEXT: s_mov_b32 s4, s0
-; GFX10-NEXT: s_mov_b32 s5, s1
-; GFX10-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, s7, s6
+; GFX10-NEXT: s_mov_b32 s0, s4
+; GFX10-NEXT: s_mov_b32 s1, s5
+; GFX10-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_shl_v2i16:
diff --git a/llvm/test/CodeGen/AMDGPU/sub.ll b/llvm/test/CodeGen/AMDGPU/sub.ll
index 6044873563254..a66226a0ef108 100644
--- a/llvm/test/CodeGen/AMDGPU/sub.ll
+++ b/llvm/test/CodeGen/AMDGPU/sub.ll
@@ -33,12 +33,12 @@ define amdgpu_kernel void @s_sub_i32(ptr addrspace(1) %out, i32 %a, i32 %b) {
;
; GFX9-LABEL: s_sub_i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_sub_i32 s2, s2, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: s_sub_i32 s0, s6, s7
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; GFX9-NEXT: s_endpgm
;
; GFX12-LABEL: s_sub_i32:
diff --git a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
index fe234a82ba6f7..02cc7d1185cf3 100644
--- a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
@@ -216,15 +216,15 @@ define amdgpu_kernel void @s_test_sub_self_v2i16(ptr addrspace(1) %out, ptr addr
define amdgpu_kernel void @s_test_sub_v2i16_kernarg(ptr addrspace(1) %out, <2 x i16> %a, <2 x i16> %b) #1 {
; GFX9-LABEL: s_test_sub_v2i16_kernarg:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX9-NEXT: s_mov_b32 s7, 0xf000
-; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX9-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-NEXT: s_mov_b32 s2, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: s_mov_b32 s4, s0
-; GFX9-NEXT: s_mov_b32 s5, s1
-; GFX9-NEXT: v_pk_sub_i16 v0, s2, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s7
+; GFX9-NEXT: s_mov_b32 s0, s4
+; GFX9-NEXT: s_mov_b32 s1, s5
+; GFX9-NEXT: v_pk_sub_i16 v0, s6, v0
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
; VI-LABEL: s_test_sub_v2i16_kernarg:
@@ -248,14 +248,14 @@ define amdgpu_kernel void @s_test_sub_v2i16_kernarg(ptr addrspace(1) %out, <2 x
;
; GFX10-LABEL: s_test_sub_v2i16_kernarg:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GFX10-NEXT: s_mov_b32 s7, 0x31016000
-; GFX10-NEXT: s_mov_b32 s6, -1
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GFX10-NEXT: s_mov_b32 s3, 0x31016000
+; GFX10-NEXT: s_mov_b32 s2, -1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_pk_sub_i16 v0, s2, s3
-; GFX10-NEXT: s_mov_b32 s4, s0
-; GFX10-NEXT: s_mov_b32 s5, s1
-; GFX10-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX10-NEXT: v_pk_sub_i16 v0, s6, s7
+; GFX10-NEXT: s_mov_b32 s0, s4
+; GFX10-NEXT: s_mov_b32 s1, s5
+; GFX10-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_test_sub_v2i16_kernarg:
diff --git a/llvm/test/CodeGen/AMDGPU/v_cndmask.ll b/llvm/test/CodeGen/AMDGPU/v_cndmask.ll
index a8f3635416cff..d1bf5ecb56984 100644
--- a/llvm/test/CodeGen/AMDGPU/v_cndmask.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_cndmask.ll
@@ -137,13 +137,13 @@ define amdgpu_kernel void @v_cnd_nan(ptr addrspace(1) %out, i32 %c, float %f) #0
;
; GFX10-LABEL: v_cnd_nan:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_cmp_eq_u32 s2, 0
-; GFX10-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v1, -1, s3, s[4:5]
-; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX10-NEXT: s_cmp_eq_u32 s6, 0
+; GFX10-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v1, -1, s7, s[0:1]
+; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: v_cnd_nan:
diff --git a/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll b/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
index 8579cbdf47137..ee99fcc586334 100644
--- a/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
@@ -104,15 +104,15 @@ define amdgpu_kernel void @basic_smax_smin_sgpr(ptr addrspace(1) %out, i32 inreg
;
; SDAG-GFX9-LABEL: basic_smax_smin_sgpr:
; SDAG-GFX9: ; %bb.0:
-; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, 0xff
; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-GFX9-NEXT: v_med3_i16 v2, s2, 0, v1
-; SDAG-GFX9-NEXT: v_med3_i16 v1, s3, 0, v1
+; SDAG-GFX9-NEXT: v_med3_i16 v2, s6, 0, v1
+; SDAG-GFX9-NEXT: v_med3_i16 v1, s7, 0, v1
; SDAG-GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
; SDAG-GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
-; SDAG-GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; SDAG-GFX9-NEXT: global_store_dword v0, v1, s[4:5]
; SDAG-GFX9-NEXT: s_endpgm
;
; SDAG-GFX11-LABEL: basic_smax_smin_sgpr:
@@ -156,22 +156,22 @@ define amdgpu_kernel void @basic_smax_smin_sgpr(ptr addrspace(1) %out, i32 inreg
;
; GISEL-GFX9-LABEL: basic_smax_smin_sgpr:
; GISEL-GFX9: ; %bb.0:
-; GISEL-GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
-; GISEL-GFX9-NEXT: s_sext_i32_i16 s4, 0
-; GISEL-GFX9-NEXT: s_sext_i32_i16 s5, 0xff
+; GISEL-GFX9-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
+; GISEL-GFX9-NEXT: s_sext_i32_i16 s0, 0
+; GISEL-GFX9-NEXT: s_sext_i32_i16 s1, 0xff
; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, 0
; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: s_sext_i32_i16 s2, s6
+; GISEL-GFX9-NEXT: s_sext_i32_i16 s3, s7
+; GISEL-GFX9-NEXT: s_max_i32 s2, s2, s0
+; GISEL-GFX9-NEXT: s_max_i32 s0, s3, s0
; GISEL-GFX9-NEXT: s_sext_i32_i16 s2, s2
-; GISEL-GFX9-NEXT: s_sext_i32_i16 s3, s3
-; GISEL-GFX9-NEXT: s_max_i32 s2, s2, s4
-; GISEL-GFX9-NEXT: s_max_i32 s3, s3, s4
-; GISEL-GFX9-NEXT: s_sext_i32_i16 s2, s2
-; GISEL-GFX9-NEXT: s_sext_i32_i16 s3, s3
-; GISEL-GFX9-NEXT: s_min_i32 s2, s2, s5
-; GISEL-GFX9-NEXT: s_min_i32 s3, s3, s5
-; GISEL-GFX9-NEXT: s_pack_ll_b32_b16 s2, s2, s3
-; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GISEL-GFX9-NEXT: global_store_dword v1, v0, s[0:1]
+; GISEL-GFX9-NEXT: s_sext_i32_i16 s0, s0
+; GISEL-GFX9-NEXT: s_min_i32 s2, s2, s1
+; GISEL-GFX9-NEXT: s_min_i32 s0, s0, s1
+; GISEL-GFX9-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX9-NEXT: global_store_dword v1, v0, s[4:5]
; GISEL-GFX9-NEXT: s_endpgm
;
; GISEL-GFX11-LABEL: basic_smax_smin_sgpr:
diff --git a/llvm/test/CodeGen/AMDGPU/wave32.ll b/llvm/test/CodeGen/AMDGPU/wave32.ll
index dae46361b9bcc..e3cfb5ecaf18e 100644
--- a/llvm/test/CodeGen/AMDGPU/wave32.ll
+++ b/llvm/test/CodeGen/AMDGPU/wave32.ll
@@ -1343,40 +1343,40 @@ exit:
define amdgpu_kernel void @fdiv_f32(ptr addrspace(1) %out, float %a, float %b) #0 {
; GFX1032-LABEL: fdiv_f32:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX1032-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_div_scale_f32 v0, s4, s3, s3, s2
+; GFX1032-NEXT: v_div_scale_f32 v0, s0, s7, s7, s6
; GFX1032-NEXT: v_rcp_f32_e32 v1, v0
; GFX1032-NEXT: v_fma_f32 v2, -v0, v1, 1.0
; GFX1032-NEXT: v_fmac_f32_e32 v1, v2, v1
-; GFX1032-NEXT: v_div_scale_f32 v2, vcc_lo, s2, s3, s2
+; GFX1032-NEXT: v_div_scale_f32 v2, vcc_lo, s6, s7, s6
; GFX1032-NEXT: v_mul_f32_e32 v3, v2, v1
; GFX1032-NEXT: v_fma_f32 v4, -v0, v3, v2
; GFX1032-NEXT: v_fmac_f32_e32 v3, v4, v1
; GFX1032-NEXT: v_fma_f32 v0, -v0, v3, v2
; GFX1032-NEXT: v_div_fmas_f32 v0, v0, v1, v3
; GFX1032-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032-NEXT: v_div_fixup_f32 v0, v0, s3, s2
-; GFX1032-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX1032-NEXT: v_div_fixup_f32 v0, v0, s7, s6
+; GFX1032-NEXT: global_store_dword v1, v0, s[4:5]
; GFX1032-NEXT: s_endpgm
;
; GFX1064-LABEL: fdiv_f32:
; GFX1064: ; %bb.0: ; %entry
-; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX1064-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_div_scale_f32 v0, s[4:5], s3, s3, s2
+; GFX1064-NEXT: v_div_scale_f32 v0, s[0:1], s7, s7, s6
; GFX1064-NEXT: v_rcp_f32_e32 v1, v0
; GFX1064-NEXT: v_fma_f32 v2, -v0, v1, 1.0
; GFX1064-NEXT: v_fmac_f32_e32 v1, v2, v1
-; GFX1064-NEXT: v_div_scale_f32 v2, vcc, s2, s3, s2
+; GFX1064-NEXT: v_div_scale_f32 v2, vcc, s6, s7, s6
; GFX1064-NEXT: v_mul_f32_e32 v3, v2, v1
; GFX1064-NEXT: v_fma_f32 v4, -v0, v3, v2
; GFX1064-NEXT: v_fmac_f32_e32 v3, v4, v1
; GFX1064-NEXT: v_fma_f32 v0, -v0, v3, v2
; GFX1064-NEXT: v_div_fmas_f32 v0, v0, v1, v3
; GFX1064-NEXT: v_mov_b32_e32 v1, 0
-; GFX1064-NEXT: v_div_fixup_f32 v0, v0, s3, s2
-; GFX1064-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX1064-NEXT: v_div_fixup_f32 v0, v0, s7, s6
+; GFX1064-NEXT: global_store_dword v1, v0, s[4:5]
; GFX1064-NEXT: s_endpgm
entry:
%fdiv = fdiv float %a, %b
@@ -2137,23 +2137,23 @@ main_body:
define amdgpu_kernel void @test_intr_fcmp_i64(ptr addrspace(1) %out, float %src, float %a) {
; GFX1032-LABEL: test_intr_fcmp_i64:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX1032-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_cmp_eq_f32_e64 s2, s2, |s3|
-; GFX1032-NEXT: v_mov_b32_e32 v0, s2
-; GFX1032-NEXT: global_store_dwordx2 v1, v[0:1], s[0:1]
+; GFX1032-NEXT: v_cmp_eq_f32_e64 s0, s6, |s7|
+; GFX1032-NEXT: v_mov_b32_e32 v0, s0
+; GFX1032-NEXT: global_store_dwordx2 v1, v[0:1], s[4:5]
; GFX1032-NEXT: s_endpgm
;
; GFX1064-LABEL: test_intr_fcmp_i64:
; GFX1064: ; %bb.0:
-; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX1064-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_cmp_eq_f32_e64 s[2:3], s2, |s3|
-; GFX1064-NEXT: v_mov_b32_e32 v0, s2
-; GFX1064-NEXT: v_mov_b32_e32 v1, s3
-; GFX1064-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX1064-NEXT: v_cmp_eq_f32_e64 s[0:1], s6, |s7|
+; GFX1064-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s1
+; GFX1064-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
; GFX1064-NEXT: s_endpgm
%temp = call float @llvm.fabs.f32(float %a)
%result = call i64 @llvm.amdgcn.fcmp.i64.f32(float %src, float %temp, i32 1)
@@ -2194,22 +2194,22 @@ define amdgpu_kernel void @test_intr_icmp_i64(ptr addrspace(1) %out, i32 %src) {
define amdgpu_kernel void @test_intr_fcmp_i32(ptr addrspace(1) %out, float %src, float %a) {
; GFX1032-LABEL: test_intr_fcmp_i32:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX1032-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX1032-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_cmp_eq_f32_e64 s2, s2, |s3|
-; GFX1032-NEXT: v_mov_b32_e32 v1, s2
-; GFX1032-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX1032-NEXT: v_cmp_eq_f32_e64 s0, s6, |s7|
+; GFX1032-NEXT: v_mov_b32_e32 v1, s0
+; GFX1032-NEXT: global_store_dword v0, v1, s[4:5]
; GFX1032-NEXT: s_endpgm
;
; GFX1064-LABEL: test_intr_fcmp_i32:
; GFX1064: ; %bb.0:
-; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x24
+; GFX1064-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x24
; GFX1064-NEXT: v_mov_b32_e32 v0, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_cmp_eq_f32_e64 s[2:3], s2, |s3|
-; GFX1064-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX1064-NEXT: v_cmp_eq_f32_e64 s[0:1], s6, |s7|
+; GFX1064-NEXT: v_mov_b32_e32 v1, s0
+; GFX1064-NEXT: global_store_dword v0, v1, s[4:5]
; GFX1064-NEXT: s_endpgm
%temp = call float @llvm.fabs.f32(float %a)
%result = call i32 @llvm.amdgcn.fcmp.i32.f32(float %src, float %temp, i32 1)
More information about the llvm-commits
mailing list