[llvm] 6636f32 - [AMDGPU] Include WWM register spill into BB Prolog (#111496)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 8 02:43:16 PDT 2024
Author: Christudasan Devadasan
Date: 2024-10-08T15:13:12+05:30
New Revision: 6636f32615000429f6391b68392e826c1de1ed64
URL: https://github.com/llvm/llvm-project/commit/6636f32615000429f6391b68392e826c1de1ed64
DIFF: https://github.com/llvm/llvm-project/commit/6636f32615000429f6391b68392e826c1de1ed64.diff
LOG: [AMDGPU] Include WWM register spill into BB Prolog (#111496)
With #93526 we split the regalloc pipeline further
to have a standalone allocation for wwm registers
and per-lane VGPRs. Currently the presence of the
wwm-spill reloads inserted at the bb-top limits the
isBasicPrologue function during the per-lane vgpr
regalloc to skip past the exec manipulation instruction
and ended up causing incorrect codegen. The wmm-spill
inserted during the wwm-regalloc pipeline should also
be included in the bb-prolog so that the per-lane vgpr
regalloc pipeline can identify the appropriate insertion
points for their spills and copies.
Added:
Modified:
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
llvm/test/CodeGen/AMDGPU/GlobalISel/image-waterfall-loop-O0.ll
llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll
llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
llvm/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll
llvm/test/CodeGen/AMDGPU/div_i128.ll
llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
llvm/test/CodeGen/AMDGPU/rem_i128.ll
llvm/test/CodeGen/AMDGPU/trap-abis.ll
llvm/test/CodeGen/AMDGPU/vgpr-spill-placement-issue61083.ll
llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll
llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 0d153df5c3977c..0c2ae382f53a19 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -8903,7 +8903,7 @@ bool SIInstrInfo::isBasicBlockPrologue(const MachineInstr &MI,
uint16_t Opcode = MI.getOpcode();
return IsNullOrVectorRegister &&
- (isSGPRSpill(Opcode) ||
+ (isSGPRSpill(Opcode) || isWWMRegSpillOpcode(Opcode) ||
(!MI.isTerminator() && Opcode != AMDGPU::COPY &&
MI.modifiesRegister(AMDGPU::EXEC, &RI)));
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/image-waterfall-loop-O0.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/image-waterfall-loop-O0.ll
index e9e7360733581a..c9426106af5dad 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/image-waterfall-loop-O0.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/image-waterfall-loop-O0.ll
@@ -68,6 +68,9 @@ define <4 x float> @waterfall_loop(<8 x i32> %vgpr_srd) {
; CHECK-NEXT: buffer_store_dword v16, off, s[0:3], s32 ; 4-byte Folded Spill
; CHECK-NEXT: s_mov_b32 exec_lo, s21
; CHECK-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: s_or_saveexec_b32 s21, -1
+; CHECK-NEXT: buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload
+; CHECK-NEXT: s_mov_b32 exec_lo, s21
; CHECK-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
@@ -84,10 +87,7 @@ define <4 x float> @waterfall_loop(<8 x i32> %vgpr_srd) {
; CHECK-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; CHECK-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; CHECK-NEXT: s_or_saveexec_b32 s21, -1
-; CHECK-NEXT: buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload
-; CHECK-NEXT: s_mov_b32 exec_lo, s21
-; CHECK-NEXT: s_waitcnt vmcnt(1)
+; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: v_readfirstlane_b32 s12, v7
; CHECK-NEXT: v_readfirstlane_b32 s10, v6
; CHECK-NEXT: v_readfirstlane_b32 s9, v5
@@ -104,7 +104,6 @@ define <4 x float> @waterfall_loop(<8 x i32> %vgpr_srd) {
; CHECK-NEXT: s_mov_b32 s17, s6
; CHECK-NEXT: s_mov_b32 s18, s5
; CHECK-NEXT: s_mov_b32 s19, s4
-; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: v_writelane_b32 v16, s12, 5
; CHECK-NEXT: v_writelane_b32 v16, s13, 6
; CHECK-NEXT: v_writelane_b32 v16, s14, 7
@@ -138,8 +137,6 @@ define <4 x float> @waterfall_loop(<8 x i32> %vgpr_srd) {
; CHECK-NEXT: buffer_store_dword v16, off, s[0:3], s32 ; 4-byte Folded Spill
; CHECK-NEXT: s_mov_b32 exec_lo, s21
; CHECK-NEXT: ; %bb.2: ; in Loop: Header=BB0_1 Depth=1
-; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; CHECK-NEXT: s_or_saveexec_b32 s21, -1
; CHECK-NEXT: buffer_load_dword v16, off, s[0:3], s32 ; 4-byte Folded Reload
; CHECK-NEXT: s_mov_b32 exec_lo, s21
@@ -157,6 +154,9 @@ define <4 x float> @waterfall_loop(<8 x i32> %vgpr_srd) {
; CHECK-NEXT: v_readlane_b32 s17, v16, 1
; CHECK-NEXT: v_readlane_b32 s18, v16, 2
; CHECK-NEXT: v_readlane_b32 s19, v16, 3
+; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: image_sample v0, v[0:1], s[8:15], s[16:19] dmask:0x1 dim:SQ_RSRC_IMG_2D
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
diff --git a/llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll b/llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll
index 2b98f61748066f..9988b2fa1eaf08 100644
--- a/llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll
@@ -29,11 +29,11 @@ define i32 @prolog_spill(i32 %arg0, i32 %arg1, i32 %arg2) {
; REGALLOC-NEXT: bb.1.Flow:
; REGALLOC-NEXT: successors: %bb.2(0x40000000), %bb.4(0x40000000)
; REGALLOC-NEXT: {{ $}}
- ; REGALLOC-NEXT: $vgpr0 = SI_SPILL_V32_RESTORE %stack.3, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.3, addrspace 5)
; REGALLOC-NEXT: $vgpr63 = SI_SPILL_WWM_V32_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
; REGALLOC-NEXT: $sgpr4 = SI_RESTORE_S32_FROM_VGPR $vgpr63, 0, implicit-def $sgpr4_sgpr5
; REGALLOC-NEXT: $sgpr5 = SI_RESTORE_S32_FROM_VGPR $vgpr63, 1
; REGALLOC-NEXT: renamable $sgpr4_sgpr5 = S_OR_SAVEEXEC_B64 killed renamable $sgpr4_sgpr5, implicit-def $exec, implicit-def dead $scc, implicit $exec
+ ; REGALLOC-NEXT: $vgpr0 = SI_SPILL_V32_RESTORE %stack.3, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.3, addrspace 5)
; REGALLOC-NEXT: SI_SPILL_V32_SAVE killed $vgpr0, %stack.6, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.6, addrspace 5)
; REGALLOC-NEXT: renamable $sgpr4_sgpr5 = S_AND_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
; REGALLOC-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr4, 2, $vgpr63, implicit-def $sgpr4_sgpr5, implicit $sgpr4_sgpr5
@@ -62,11 +62,11 @@ define i32 @prolog_spill(i32 %arg0, i32 %arg1, i32 %arg2) {
; REGALLOC-NEXT: S_BRANCH %bb.1
; REGALLOC-NEXT: {{ $}}
; REGALLOC-NEXT: bb.4.bb.3:
- ; REGALLOC-NEXT: $vgpr0 = SI_SPILL_V32_RESTORE %stack.6, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.6, addrspace 5)
; REGALLOC-NEXT: $vgpr63 = SI_SPILL_WWM_V32_RESTORE %stack.2, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.2, addrspace 5)
; REGALLOC-NEXT: $sgpr4 = SI_RESTORE_S32_FROM_VGPR $vgpr63, 2, implicit-def $sgpr4_sgpr5
; REGALLOC-NEXT: $sgpr5 = SI_RESTORE_S32_FROM_VGPR killed $vgpr63, 3
; REGALLOC-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
+ ; REGALLOC-NEXT: $vgpr0 = SI_SPILL_V32_RESTORE %stack.6, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.6, addrspace 5)
; REGALLOC-NEXT: renamable $vgpr0 = V_LSHL_ADD_U32_e64 killed $vgpr0, 2, $vgpr0, implicit $exec
; REGALLOC-NEXT: SI_RETURN implicit killed $vgpr0
bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll b/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
index 7cec15ea5be87a..75d0b83a024ff5 100644
--- a/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
+++ b/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
@@ -67,7 +67,6 @@ define amdgpu_kernel void @simple_nested_if(ptr addrspace(1) nocapture %arg) {
; GCN-O0-NEXT: s_mov_b64 exec, s[0:1]
; GCN-O0-NEXT: s_cbranch_execz .LBB0_4
; GCN-O0-NEXT: ; %bb.1: ; %bb.outer.then
-; GCN-O0-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_or_saveexec_b64 s[8:9], -1
; GCN-O0-NEXT: s_waitcnt expcnt(0)
; GCN-O0-NEXT: buffer_load_dword v4, off, s[12:15], 0 ; 4-byte Folded Reload
@@ -75,12 +74,14 @@ define amdgpu_kernel void @simple_nested_if(ptr addrspace(1) nocapture %arg) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v4, 0
; GCN-O0-NEXT: v_readlane_b32 s5, v4, 1
+; GCN-O0-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_mov_b32 s2, 0xf000
; GCN-O0-NEXT: s_mov_b32 s0, 0
; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
; GCN-O0-NEXT: s_mov_b32 s1, s2
; GCN-O0-NEXT: ; kill: def $sgpr4_sgpr5 killed $sgpr4_sgpr5 def $sgpr4_sgpr5_sgpr6_sgpr7
; GCN-O0-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_ashrrev_i32_e64 v3, 31, v0
; GCN-O0-NEXT: v_mov_b32_e32 v1, v0
; GCN-O0-NEXT: v_mov_b32_e32 v2, v3
@@ -99,8 +100,6 @@ define amdgpu_kernel void @simple_nested_if(ptr addrspace(1) nocapture %arg) {
; GCN-O0-NEXT: s_mov_b64 exec, s[0:1]
; GCN-O0-NEXT: s_cbranch_execz .LBB0_3
; GCN-O0-NEXT: ; %bb.2: ; %bb.inner.then
-; GCN-O0-NEXT: s_waitcnt expcnt(1)
-; GCN-O0-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_or_saveexec_b64 s[8:9], -1
; GCN-O0-NEXT: s_waitcnt expcnt(0)
; GCN-O0-NEXT: buffer_load_dword v4, off, s[12:15], 0 ; 4-byte Folded Reload
@@ -108,7 +107,9 @@ define amdgpu_kernel void @simple_nested_if(ptr addrspace(1) nocapture %arg) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s0, v4, 0
; GCN-O0-NEXT: v_readlane_b32 s1, v4, 1
+; GCN-O0-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: v_mov_b32_e32 v0, 1
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_add_i32_e64 v1, s[2:3], v1, v0
; GCN-O0-NEXT: v_ashrrev_i32_e64 v3, 31, v1
; GCN-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
@@ -235,7 +236,6 @@ define amdgpu_kernel void @uncollapsable_nested_if(ptr addrspace(1) nocapture %a
; GCN-O0-NEXT: s_mov_b64 exec, s[0:1]
; GCN-O0-NEXT: s_cbranch_execz .LBB1_3
; GCN-O0-NEXT: ; %bb.1: ; %bb.outer.then
-; GCN-O0-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_or_saveexec_b64 s[8:9], -1
; GCN-O0-NEXT: s_waitcnt expcnt(0)
; GCN-O0-NEXT: buffer_load_dword v4, off, s[12:15], 0 ; 4-byte Folded Reload
@@ -243,12 +243,14 @@ define amdgpu_kernel void @uncollapsable_nested_if(ptr addrspace(1) nocapture %a
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v4, 0
; GCN-O0-NEXT: v_readlane_b32 s5, v4, 1
+; GCN-O0-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_mov_b32 s2, 0xf000
; GCN-O0-NEXT: s_mov_b32 s0, 0
; GCN-O0-NEXT: ; kill: def $sgpr0 killed $sgpr0 def $sgpr0_sgpr1
; GCN-O0-NEXT: s_mov_b32 s1, s2
; GCN-O0-NEXT: ; kill: def $sgpr4_sgpr5 killed $sgpr4_sgpr5 def $sgpr4_sgpr5_sgpr6_sgpr7
; GCN-O0-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_ashrrev_i32_e64 v3, 31, v0
; GCN-O0-NEXT: v_mov_b32_e32 v1, v0
; GCN-O0-NEXT: v_mov_b32_e32 v2, v3
@@ -267,8 +269,6 @@ define amdgpu_kernel void @uncollapsable_nested_if(ptr addrspace(1) nocapture %a
; GCN-O0-NEXT: s_mov_b64 exec, s[0:1]
; GCN-O0-NEXT: s_cbranch_execz .LBB1_4
; GCN-O0-NEXT: ; %bb.2: ; %bb.inner.then
-; GCN-O0-NEXT: s_waitcnt expcnt(1)
-; GCN-O0-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_or_saveexec_b64 s[8:9], -1
; GCN-O0-NEXT: s_waitcnt expcnt(0)
; GCN-O0-NEXT: buffer_load_dword v4, off, s[12:15], 0 ; 4-byte Folded Reload
@@ -276,7 +276,9 @@ define amdgpu_kernel void @uncollapsable_nested_if(ptr addrspace(1) nocapture %a
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s0, v4, 0
; GCN-O0-NEXT: v_readlane_b32 s1, v4, 1
+; GCN-O0-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: v_mov_b32_e32 v0, 1
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_add_i32_e64 v1, s[2:3], v1, v0
; GCN-O0-NEXT: v_ashrrev_i32_e64 v3, 31, v1
; GCN-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
@@ -302,8 +304,6 @@ define amdgpu_kernel void @uncollapsable_nested_if(ptr addrspace(1) nocapture %a
; GCN-O0-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN-O0-NEXT: s_branch .LBB1_5
; GCN-O0-NEXT: .LBB1_4: ; %bb.inner.end
-; GCN-O0-NEXT: s_waitcnt expcnt(1)
-; GCN-O0-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_or_saveexec_b64 s[8:9], -1
; GCN-O0-NEXT: s_waitcnt expcnt(0)
; GCN-O0-NEXT: buffer_load_dword v4, off, s[12:15], 0 ; 4-byte Folded Reload
@@ -314,7 +314,9 @@ define amdgpu_kernel void @uncollapsable_nested_if(ptr addrspace(1) nocapture %a
; GCN-O0-NEXT: s_or_b64 exec, exec, s[2:3]
; GCN-O0-NEXT: v_readlane_b32 s0, v4, 0
; GCN-O0-NEXT: v_readlane_b32 s1, v4, 1
+; GCN-O0-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: v_mov_b32_e32 v0, 2
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_add_i32_e64 v1, s[2:3], v1, v0
; GCN-O0-NEXT: v_ashrrev_i32_e64 v3, 31, v1
; GCN-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
@@ -454,18 +456,17 @@ define amdgpu_kernel void @nested_if_if_else(ptr addrspace(1) nocapture %arg) {
; GCN-O0-NEXT: s_mov_b64 exec, s[0:1]
; GCN-O0-NEXT: s_cbranch_execz .LBB2_6
; GCN-O0-NEXT: ; %bb.1: ; %bb.outer.then
-; GCN-O0-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_or_saveexec_b64 s[6:7], -1
; GCN-O0-NEXT: s_waitcnt expcnt(0)
; GCN-O0-NEXT: buffer_load_dword v4, off, s[12:15], 0 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_mov_b64 exec, s[6:7]
+; GCN-O0-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_mov_b32 s0, 2
-; GCN-O0-NEXT: s_waitcnt vmcnt(1)
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_cmp_ne_u32_e64 s[0:1], v0, s0
; GCN-O0-NEXT: s_mov_b64 s[2:3], exec
; GCN-O0-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1]
; GCN-O0-NEXT: s_xor_b64 s[2:3], s[0:1], s[2:3]
-; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_writelane_b32 v4, s2, 4
; GCN-O0-NEXT: v_writelane_b32 v4, s3, 5
; GCN-O0-NEXT: s_or_saveexec_b64 s[6:7], -1
@@ -492,7 +493,6 @@ define amdgpu_kernel void @nested_if_if_else(ptr addrspace(1) nocapture %arg) {
; GCN-O0-NEXT: s_xor_b64 exec, exec, s[0:1]
; GCN-O0-NEXT: s_cbranch_execz .LBB2_5
; GCN-O0-NEXT: ; %bb.3: ; %bb.then
-; GCN-O0-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_or_saveexec_b64 s[6:7], -1
; GCN-O0-NEXT: s_waitcnt expcnt(0)
; GCN-O0-NEXT: buffer_load_dword v4, off, s[12:15], 0 ; 4-byte Folded Reload
@@ -500,7 +500,9 @@ define amdgpu_kernel void @nested_if_if_else(ptr addrspace(1) nocapture %arg) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s0, v4, 0
; GCN-O0-NEXT: v_readlane_b32 s1, v4, 1
+; GCN-O0-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: v_mov_b32_e32 v0, 1
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_add_i32_e64 v1, s[2:3], v1, v0
; GCN-O0-NEXT: v_ashrrev_i32_e64 v3, 31, v1
; GCN-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
@@ -516,7 +518,6 @@ define amdgpu_kernel void @nested_if_if_else(ptr addrspace(1) nocapture %arg) {
; GCN-O0-NEXT: buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
; GCN-O0-NEXT: s_branch .LBB2_5
; GCN-O0-NEXT: .LBB2_4: ; %bb.else
-; GCN-O0-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_or_saveexec_b64 s[6:7], -1
; GCN-O0-NEXT: s_waitcnt expcnt(0)
; GCN-O0-NEXT: buffer_load_dword v4, off, s[12:15], 0 ; 4-byte Folded Reload
@@ -524,7 +525,9 @@ define amdgpu_kernel void @nested_if_if_else(ptr addrspace(1) nocapture %arg) {
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s0, v4, 0
; GCN-O0-NEXT: v_readlane_b32 s1, v4, 1
+; GCN-O0-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: v_mov_b32_e32 v0, 2
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_add_i32_e64 v1, s[2:3], v1, v0
; GCN-O0-NEXT: v_ashrrev_i32_e64 v3, 31, v1
; GCN-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
@@ -721,13 +724,13 @@ define amdgpu_kernel void @nested_if_else_if(ptr addrspace(1) nocapture %arg) {
; GCN-O0-NEXT: s_xor_b64 exec, exec, s[0:1]
; GCN-O0-NEXT: s_cbranch_execz .LBB3_8
; GCN-O0-NEXT: ; %bb.2: ; %bb.outer.then
-; GCN-O0-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:12 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v2, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v3, off, s[12:15], 0 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_or_saveexec_b64 s[8:9], -1
; GCN-O0-NEXT: s_waitcnt expcnt(0)
; GCN-O0-NEXT: buffer_load_dword v6, off, s[12:15], 0 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_mov_b64 exec, s[8:9]
+; GCN-O0-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:12 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v2, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v3, off, s[12:15], 0 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_mov_b32 s0, 0xf000
; GCN-O0-NEXT: s_mov_b32 s2, 0
; GCN-O0-NEXT: s_mov_b32 s4, s2
@@ -737,12 +740,11 @@ define amdgpu_kernel void @nested_if_else_if(ptr addrspace(1) nocapture %arg) {
; GCN-O0-NEXT: ; kill: def $sgpr0_sgpr1 killed $sgpr0_sgpr1 def $sgpr0_sgpr1_sgpr2_sgpr3
; GCN-O0-NEXT: s_mov_b64 s[2:3], s[4:5]
; GCN-O0-NEXT: v_mov_b32_e32 v1, 1
-; GCN-O0-NEXT: s_waitcnt vmcnt(1)
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: buffer_store_dword v1, v[2:3], s[0:3], 0 addr64 offset:4
; GCN-O0-NEXT: s_mov_b32 s0, 2
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[2:3], v0, s0
; GCN-O0-NEXT: s_mov_b64 s[0:1], exec
-; GCN-O0-NEXT: s_waitcnt vmcnt(1)
; GCN-O0-NEXT: v_writelane_b32 v6, s0, 4
; GCN-O0-NEXT: v_writelane_b32 v6, s1, 5
; GCN-O0-NEXT: s_or_saveexec_b64 s[8:9], -1
@@ -768,13 +770,13 @@ define amdgpu_kernel void @nested_if_else_if(ptr addrspace(1) nocapture %arg) {
; GCN-O0-NEXT: buffer_store_dword v0, v[1:2], s[0:3], 0 addr64 offset:8
; GCN-O0-NEXT: s_branch .LBB3_7
; GCN-O0-NEXT: .LBB3_4: ; %bb.outer.else
-; GCN-O0-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:12 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v2, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
-; GCN-O0-NEXT: buffer_load_dword v3, off, s[12:15], 0 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_or_saveexec_b64 s[8:9], -1
; GCN-O0-NEXT: s_waitcnt expcnt(0)
; GCN-O0-NEXT: buffer_load_dword v6, off, s[12:15], 0 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_mov_b64 exec, s[8:9]
+; GCN-O0-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:12 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v2, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v3, off, s[12:15], 0 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_mov_b32 s1, 0xf000
; GCN-O0-NEXT: s_mov_b32 s0, 0
; GCN-O0-NEXT: s_mov_b32 s2, s0
@@ -784,11 +786,10 @@ define amdgpu_kernel void @nested_if_else_if(ptr addrspace(1) nocapture %arg) {
; GCN-O0-NEXT: ; kill: def $sgpr4_sgpr5 killed $sgpr4_sgpr5 def $sgpr4_sgpr5_sgpr6_sgpr7
; GCN-O0-NEXT: s_mov_b64 s[6:7], s[2:3]
; GCN-O0-NEXT: v_mov_b32_e32 v1, 3
-; GCN-O0-NEXT: s_waitcnt vmcnt(1)
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: buffer_store_dword v1, v[2:3], s[4:7], 0 addr64 offset:12
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[2:3], v0, s0
; GCN-O0-NEXT: s_mov_b64 s[0:1], exec
-; GCN-O0-NEXT: s_waitcnt vmcnt(1)
; GCN-O0-NEXT: v_writelane_b32 v6, s0, 6
; GCN-O0-NEXT: v_writelane_b32 v6, s1, 7
; GCN-O0-NEXT: s_or_saveexec_b64 s[8:9], -1
@@ -926,7 +927,6 @@ define amdgpu_kernel void @s_endpgm_unsafe_barrier(ptr addrspace(1) nocapture %a
; GCN-O0-NEXT: s_mov_b64 exec, s[0:1]
; GCN-O0-NEXT: s_cbranch_execz .LBB4_2
; GCN-O0-NEXT: ; %bb.1: ; %bb.then
-; GCN-O0-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_or_saveexec_b64 s[6:7], -1
; GCN-O0-NEXT: s_waitcnt expcnt(0)
; GCN-O0-NEXT: buffer_load_dword v3, off, s[12:15], 0 ; 4-byte Folded Reload
@@ -934,12 +934,14 @@ define amdgpu_kernel void @s_endpgm_unsafe_barrier(ptr addrspace(1) nocapture %a
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s0, v3, 0
; GCN-O0-NEXT: v_readlane_b32 s1, v3, 1
+; GCN-O0-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_mov_b32 s2, 0xf000
; GCN-O0-NEXT: s_mov_b32 s4, 0
; GCN-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
; GCN-O0-NEXT: s_mov_b32 s5, s2
; GCN-O0-NEXT: ; kill: def $sgpr0_sgpr1 killed $sgpr0_sgpr1 def $sgpr0_sgpr1_sgpr2_sgpr3
; GCN-O0-NEXT: s_mov_b64 s[2:3], s[4:5]
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_ashrrev_i32_e64 v2, 31, v0
; GCN-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GCN-O0-NEXT: v_mov_b32_e32 v1, v2
@@ -1064,8 +1066,6 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-O0-NEXT: s_mov_b64 exec, s[14:15]
; GCN-O0-NEXT: .LBB5_1: ; %bb1
; GCN-O0-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN-O0-NEXT: s_waitcnt expcnt(1)
-; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
; GCN-O0-NEXT: s_waitcnt expcnt(0)
; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
@@ -1077,7 +1077,9 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-O0-NEXT: v_readlane_b32 s7, v6, 1
; GCN-O0-NEXT: v_writelane_b32 v6, s6, 4
; GCN-O0-NEXT: v_writelane_b32 v6, s7, 5
+; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_mov_b32 s4, 0x207
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_cmp_lt_i32_e64 s[4:5], v0, s4
; GCN-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
; GCN-O0-NEXT: v_writelane_b32 v6, s4, 6
@@ -1094,7 +1096,6 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-O0-NEXT: s_cbranch_execnz .LBB5_1
; GCN-O0-NEXT: ; %bb.2: ; %bb2
; GCN-O0-NEXT: ; in Loop: Header=BB5_1 Depth=1
-; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
; GCN-O0-NEXT: s_waitcnt expcnt(0)
; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
@@ -1103,7 +1104,9 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-O0-NEXT: v_readlane_b32 s4, v6, 6
; GCN-O0-NEXT: v_readlane_b32 s5, v6, 7
; GCN-O0-NEXT: s_or_b64 exec, exec, s[4:5]
+; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_mov_b32 s6, 0
+; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_cmp_ne_u32_e64 s[4:5], v0, s6
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v0, s6
; GCN-O0-NEXT: v_writelane_b32 v6, s4, 8
@@ -1188,46 +1191,48 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-O0-NEXT: s_branch .LBB5_6
; GCN-O0-NEXT: .LBB5_5: ; %Flow2
; GCN-O0-NEXT: ; in Loop: Header=BB5_1 Depth=1
-; GCN-O0-NEXT: s_waitcnt expcnt(3)
-; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt expcnt(2)
-; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt expcnt(1)
-; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt expcnt(0)
-; GCN-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
+; GCN-O0-NEXT: s_waitcnt expcnt(0)
; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_mov_b64 exec, s[14:15]
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v6, 10
; GCN-O0-NEXT: v_readlane_b32 s5, v6, 11
; GCN-O0-NEXT: s_or_b64 exec, exec, s[4:5]
+; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; GCN-O0-NEXT: s_waitcnt vmcnt(3)
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:40 ; 4-byte Folded Spill
+; GCN-O0-NEXT: s_waitcnt vmcnt(3)
; GCN-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:44 ; 4-byte Folded Spill
+; GCN-O0-NEXT: s_waitcnt vmcnt(3)
; GCN-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:48 ; 4-byte Folded Spill
+; GCN-O0-NEXT: s_waitcnt vmcnt(3)
; GCN-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:52 ; 4-byte Folded Spill
; GCN-O0-NEXT: s_branch .LBB5_7
; GCN-O0-NEXT: .LBB5_6: ; %Flow
; GCN-O0-NEXT: ; in Loop: Header=BB5_1 Depth=1
-; GCN-O0-NEXT: s_waitcnt expcnt(3)
-; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt expcnt(2)
-; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt expcnt(1)
-; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt expcnt(0)
-; GCN-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
+; GCN-O0-NEXT: s_waitcnt expcnt(0)
; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_mov_b64 exec, s[14:15]
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_readlane_b32 s4, v6, 12
; GCN-O0-NEXT: v_readlane_b32 s5, v6, 13
; GCN-O0-NEXT: s_or_b64 exec, exec, s[4:5]
+; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GCN-O0-NEXT: s_waitcnt vmcnt(3)
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
+; GCN-O0-NEXT: s_waitcnt vmcnt(3)
; GCN-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
+; GCN-O0-NEXT: s_waitcnt vmcnt(3)
; GCN-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
+; GCN-O0-NEXT: s_waitcnt vmcnt(3)
; GCN-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; GCN-O0-NEXT: s_branch .LBB5_5
; GCN-O0-NEXT: .LBB5_7: ; %bb10
@@ -1266,14 +1271,6 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-O0-NEXT: s_mov_b64 exec, s[14:15]
; GCN-O0-NEXT: .LBB5_9: ; %Flow3
; GCN-O0-NEXT: ; in Loop: Header=BB5_1 Depth=1
-; GCN-O0-NEXT: s_waitcnt expcnt(4)
-; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt expcnt(3)
-; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt expcnt(2)
-; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_waitcnt expcnt(1)
-; GCN-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
; GCN-O0-NEXT: s_waitcnt expcnt(0)
; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
@@ -1286,6 +1283,10 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-O0-NEXT: v_readlane_b32 s7, v6, 5
; GCN-O0-NEXT: v_readlane_b32 s4, v6, 14
; GCN-O0-NEXT: v_readlane_b32 s5, v6, 15
+; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_and_b64 s[4:5], exec, s[4:5]
; GCN-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
; GCN-O0-NEXT: s_mov_b64 s[6:7], 0
@@ -1300,9 +1301,13 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
; GCN-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
; GCN-O0-NEXT: s_mov_b64 exec, s[14:15]
+; GCN-O0-NEXT: s_waitcnt vmcnt(4)
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
+; GCN-O0-NEXT: s_waitcnt vmcnt(4)
; GCN-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
+; GCN-O0-NEXT: s_waitcnt vmcnt(4)
; GCN-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
+; GCN-O0-NEXT: s_waitcnt vmcnt(4)
; GCN-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; GCN-O0-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN-O0-NEXT: s_cbranch_execnz .LBB5_1
diff --git a/llvm/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll b/llvm/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll
index 7c09fec908f93e..08644572372c36 100644
--- a/llvm/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll
@@ -46,9 +46,6 @@
; VMEM: [[ENDIF]]:
-; Restore val
-; VGPR: buffer_load_dword [[RELOAD_VAL:v[0-9]+]], off, s[0:3], 0 offset:[[VAL_OFFSET]] ; 4-byte Folded Reload
-
; Reload and restore exec mask
; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_LO_LANE]]
; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_HI_LANE]]
@@ -61,7 +58,7 @@
; GCN: s_or_b64 exec, exec, s[[[S_RELOAD_SAVEEXEC_LO]]:[[S_RELOAD_SAVEEXEC_HI]]]
; Restore val
-; VMEM: buffer_load_dword [[RELOAD_VAL:v[0-9]+]], off, s[0:3], 0 offset:[[VAL_OFFSET]] ; 4-byte Folded Reload
+; GCN: buffer_load_dword [[RELOAD_VAL:v[0-9]+]], off, s[0:3], 0 offset:[[VAL_OFFSET]] ; 4-byte Folded Reload
; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RELOAD_VAL]]
@@ -123,7 +120,6 @@ endif:
; GCN: buffer_store_dword v[[VAL_LOOP_RELOAD]], off, s[0:3], 0 offset:[[VAL_SUB_OFFSET:[0-9]+]] ; 4-byte Folded Spill
; GCN: [[END]]:
-; VGPR: buffer_load_dword v[[VAL_END:[0-9]+]], off, s[0:3], 0 offset:[[VAL_SUB_OFFSET]] ; 4-byte Folded Reload
; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_LO_LANE]]
; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_HI_LANE]]
@@ -134,7 +130,7 @@ endif:
; GCN: s_or_b64 exec, exec, s[[[S_RELOAD_SAVEEXEC_LO]]:[[S_RELOAD_SAVEEXEC_HI]]]
-; VMEM: buffer_load_dword v[[VAL_END:[0-9]+]], off, s[0:3], 0 offset:[[VAL_SUB_OFFSET]] ; 4-byte Folded Reload
+; GCN: buffer_load_dword v[[VAL_END:[0-9]+]], off, s[0:3], 0 offset:[[VAL_SUB_OFFSET]] ; 4-byte Folded Reload
; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v[[VAL_END]]
@@ -194,7 +190,6 @@ end:
; GCN-NEXT: s_branch [[ELSE:.LBB[0-9]+_[0-9]+]]
; GCN: [[FLOW]]: ; %Flow
-; VGPR: buffer_load_dword [[FLOW_VAL:v[0-9]+]], off, s[0:3], 0 offset:[[FLOW_VAL_OFFSET:[0-9]+]] ; 4-byte Folded Reload
; VGPR: buffer_load_dword [[SPILL_VGPR:v[0-9]+]], off, s[0:3], 0 ; 4-byte Folded Reload
; VGPR: v_readlane_b32 s[[FLOW_S_RELOAD_SAVEEXEC_LO:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_LO_LANE]]
; VGPR: v_readlane_b32 s[[FLOW_S_RELOAD_SAVEEXEC_HI:[0-9]+]], [[SPILL_VGPR]], [[SAVEEXEC_HI_LANE]]
@@ -206,7 +201,7 @@ end:
; GCN: s_or_saveexec_b64 s[[[FLOW_S_RELOAD_SAVEEXEC_LO_SAVEEXEC:[0-9]+]]:[[FLOW_S_RELOAD_SAVEEXEC_HI_SAVEEXEC:[0-9]+]]], s[[[FLOW_S_RELOAD_SAVEEXEC_LO]]:[[FLOW_S_RELOAD_SAVEEXEC_HI]]]
-; VMEM: buffer_load_dword [[FLOW_VAL:v[0-9]+]], off, s[0:3], 0 offset:[[FLOW_VAL_OFFSET:[0-9]+]] ; 4-byte Folded Reload
+; GCN: buffer_load_dword [[FLOW_VAL:v[0-9]+]], off, s[0:3], 0 offset:[[FLOW_VAL_OFFSET:[0-9]+]] ; 4-byte Folded Reload
; Regular spill value restored after exec modification
; Followed by spill
@@ -240,7 +235,6 @@ end:
; GCN-NEXT: s_branch [[FLOW]]
; GCN: [[ENDIF]]:
-; VGPR: buffer_load_dword v[[RESULT:[0-9]+]], off, s[0:3], 0 offset:[[RESULT_OFFSET]] ; 4-byte Folded Reload
; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_LO:[0-9]+]], [[SPILL_VGPR]], [[FLOW_SAVEEXEC_LO_LANE]]
; VGPR: v_readlane_b32 s[[S_RELOAD_SAVEEXEC_HI:[0-9]+]], [[SPILL_VGPR]], [[FLOW_SAVEEXEC_HI_LANE]]
@@ -252,7 +246,7 @@ end:
; GCN: s_or_b64 exec, exec, s[[[S_RELOAD_SAVEEXEC_LO]]:[[S_RELOAD_SAVEEXEC_HI]]]
-; VMEM: buffer_load_dword v[[RESULT:[0-9]+]], off, s[0:3], 0 offset:[[RESULT_OFFSET]] ; 4-byte Folded Reload
+; GCN: buffer_load_dword v[[RESULT:[0-9]+]], off, s[0:3], 0 offset:[[RESULT_OFFSET]] ; 4-byte Folded Reload
; GCN: flat_store_dword v{{\[[0-9]+:[0-9]+\]}}, v[[RESULT]]
define amdgpu_kernel void @divergent_if_else_endif(ptr addrspace(1) %out) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/div_i128.ll b/llvm/test/CodeGen/AMDGPU/div_i128.ll
index 6686742e449f5c..d94ec56842ab87 100644
--- a/llvm/test/CodeGen/AMDGPU/div_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_i128.ll
@@ -581,10 +581,6 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:92 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB0_5
; GFX9-O0-NEXT: .LBB0_3: ; %Flow2
-; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23]
@@ -592,8 +588,13 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 4
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 5
; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
@@ -640,6 +641,13 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB0_3
; GFX9-O0-NEXT: .LBB0_5: ; %Flow1
+; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
+; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 8
+; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 9
+; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
@@ -648,15 +656,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
-; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23]
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 8
-; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 9
-; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
@@ -670,6 +672,12 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_branch .LBB0_4
; GFX9-O0-NEXT: .LBB0_6: ; %udiv-do-while
; GFX9-O0-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
+; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: v_readlane_b32 s6, v30, 10
+; GFX9-O0-NEXT: v_readlane_b32 s7, v30, 11
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:208 ; 4-byte Folded Reload
@@ -694,13 +702,8 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:276 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
-; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23]
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT: v_readlane_b32 s6, v30, 10
-; GFX9-O0-NEXT: v_readlane_b32 s7, v30, 11
; GFX9-O0-NEXT: s_mov_b32 s4, 63
+; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: v_lshrrev_b64 v[28:29], s4, v[2:3]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v29
; GFX9-O0-NEXT: s_mov_b32 s5, 1
@@ -727,6 +730,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v1
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v29
+; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
; GFX9-O0-NEXT: v_mov_b32_e32 v10, v27
; GFX9-O0-NEXT: v_or3_b32 v6, v6, v7, v10
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0
@@ -736,6 +740,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v3
+; GFX9-O0-NEXT: s_waitcnt vmcnt(8)
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v25
; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v7
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2
@@ -747,10 +752,12 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v10, v5
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v22
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v23
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v13, v11
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v14
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v15
; GFX9-O0-NEXT: v_sub_co_u32_e32 v13, vcc, v13, v6
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_subb_co_u32_e32 v12, vcc, v12, v10, vcc
; GFX9-O0-NEXT: v_subb_co_u32_e32 v11, vcc, v11, v4, vcc
; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v7, v5, vcc
@@ -890,6 +897,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_cbranch_execnz .LBB0_6
; GFX9-O0-NEXT: s_branch .LBB0_1
; GFX9-O0-NEXT: .LBB0_7: ; %udiv-preheader
+; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
+; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23]
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
@@ -906,12 +916,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
-; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23]
-; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], v4, v[20:21]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-O0-NEXT: s_mov_b32 s6, 64
@@ -992,7 +999,6 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v14, s8
; GFX9-O0-NEXT: v_mov_b32_e32 v13, s7
; GFX9-O0-NEXT: v_mov_b32_e32 v12, s6
-; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-O0-NEXT: v_writelane_b32 v30, s4, 10
; GFX9-O0-NEXT: v_writelane_b32 v30, s5, 11
; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
@@ -1024,6 +1030,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:204 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB0_6
; GFX9-O0-NEXT: .LBB0_8: ; %udiv-bb1
+; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
+; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23]
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
@@ -1032,12 +1041,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
-; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23]
; GFX9-O0-NEXT: s_mov_b64 s[6:7], 1
; GFX9-O0-NEXT: s_mov_b32 s5, s6
-; GFX9-O0-NEXT: s_waitcnt vmcnt(2)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v0
; GFX9-O0-NEXT: s_mov_b32 s4, s7
; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0
@@ -1048,7 +1054,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s5
; GFX9-O0-NEXT: v_add_co_u32_e32 v8, vcc, v3, v4
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v4, vcc
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s8
; GFX9-O0-NEXT: v_addc_co_u32_e32 v0, vcc, v0, v4, vcc
@@ -1152,7 +1158,6 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_mov_b64 s[6:7], exec
; GFX9-O0-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5]
; GFX9-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7]
-; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 8
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 9
; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
@@ -1710,10 +1715,6 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:112 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB0_5
; GFX9-G-O0-NEXT: .LBB0_3: ; %Flow2
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-G-O0-NEXT: buffer_load_dword v34, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
@@ -1721,10 +1722,17 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_readlane_b32 s4, v34, 0
; GFX9-G-O0-NEXT: v_readlane_b32 s5, v34, 1
; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:172 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB0_9
; GFX9-G-O0-NEXT: .LBB0_4: ; %udiv-loop-exit
@@ -1784,6 +1792,13 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB0_3
; GFX9-G-O0-NEXT: .LBB0_5: ; %Flow1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: buffer_load_dword v34, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-G-O0-NEXT: v_readlane_b32 s4, v34, 4
+; GFX9-G-O0-NEXT: v_readlane_b32 s5, v34, 5
+; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
@@ -1792,17 +1807,13 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
-; GFX9-G-O0-NEXT: buffer_load_dword v34, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_readlane_b32 s4, v34, 4
-; GFX9-G-O0-NEXT: v_readlane_b32 s5, v34, 5
-; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
@@ -1812,6 +1823,12 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_branch .LBB0_4
; GFX9-G-O0-NEXT: .LBB0_6: ; %udiv-do-while
; GFX9-G-O0-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: buffer_load_dword v34, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-G-O0-NEXT: v_readlane_b32 s6, v34, 6
+; GFX9-G-O0-NEXT: v_readlane_b32 s7, v34, 7
; GFX9-G-O0-NEXT: buffer_load_dword v22, off, s[0:3], s32 offset:212 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v23, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
@@ -1836,15 +1853,11 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:280 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
-; GFX9-G-O0-NEXT: buffer_load_dword v34, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_readlane_b32 s6, v34, 6
-; GFX9-G-O0-NEXT: v_readlane_b32 s7, v34, 7
; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(18)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v2
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v3
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v5
; GFX9-G-O0-NEXT: s_mov_b32 s8, 1
@@ -1894,8 +1907,10 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s8
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(10)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v28, v30
; GFX9-G-O0-NEXT: v_mov_b32_e32 v29, v31
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(8)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v24, v32
; GFX9-G-O0-NEXT: v_mov_b32_e32 v25, v33
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v28
@@ -1915,6 +1930,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: ; kill: def $vgpr0_vgpr1 killed $vgpr0_vgpr1 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v14
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v15
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v13, s[8:9], v13, v4
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v12, s[8:9], v12, v9, s[8:9]
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[8:9], v10, v7, s[8:9]
@@ -2027,6 +2043,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_cbranch_execnz .LBB0_6
; GFX9-G-O0-NEXT: s_branch .LBB0_1
; GFX9-G-O0-NEXT: .LBB0_7: ; %udiv-preheader
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: buffer_load_dword v34, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
@@ -2044,15 +2063,12 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
-; GFX9-G-O0-NEXT: buffer_load_dword v34, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-G-O0-NEXT: s_mov_b32 s4, 64
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(4)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v22, v17
; GFX9-G-O0-NEXT: v_mov_b32_e32 v23, v16
; GFX9-G-O0-NEXT: ; kill: def $vgpr20 killed $vgpr20 def $vgpr20_vgpr21 killed $exec
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(2)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s4
; GFX9-G-O0-NEXT: v_sub_u32_e64 v4, v18, v4
@@ -2063,7 +2079,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_cmp_lt_u32_e64 s[4:5], v18, v6
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, s6
; GFX9-G-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v18, v6
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-G-O0-NEXT: v_lshrrev_b64 v[6:7], v18, v[20:21]
; GFX9-G-O0-NEXT: v_lshrrev_b64 v[25:26], v18, v[22:23]
; GFX9-G-O0-NEXT: v_lshlrev_b64 v[23:24], v5, v[20:21]
@@ -2112,7 +2128,6 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], s[8:9]
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-G-O0-NEXT: v_writelane_b32 v34, s8, 6
; GFX9-G-O0-NEXT: v_writelane_b32 v34, s9, 7
; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
@@ -2144,6 +2159,9 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:224 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB0_6
; GFX9-G-O0-NEXT: .LBB0_8: ; %udiv-bb1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
+; GFX9-G-O0-NEXT: buffer_load_dword v34, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
@@ -2152,20 +2170,17 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
-; GFX9-G-O0-NEXT: buffer_load_dword v34, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0
; GFX9-G-O0-NEXT: s_mov_b32 s6, 1
; GFX9-G-O0-NEXT: s_mov_b32 s10, 0
; GFX9-G-O0-NEXT: s_mov_b32 s9, 0
; GFX9-G-O0-NEXT: s_mov_b32 s8, 0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s6
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(4)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-G-O0-NEXT: v_add_co_u32_e64 v4, s[6:7], v2, v4
; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:324 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s10
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(2)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[6:7], v5, v7, s[6:7]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v7, s9
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v7, s[6:7], v6, v7, s[6:7]
@@ -2257,7 +2272,6 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], exec
; GFX9-G-O0-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5]
; GFX9-G-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7]
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(17)
; GFX9-G-O0-NEXT: v_writelane_b32 v34, s6, 4
; GFX9-G-O0-NEXT: v_writelane_b32 v34, s7, 5
; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
@@ -2761,10 +2775,6 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB1_5
; GFX9-O0-NEXT: .LBB1_3: ; %Flow2
-; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
@@ -2772,8 +2782,13 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 2
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 3
; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
@@ -2820,6 +2835,13 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB1_3
; GFX9-O0-NEXT: .LBB1_5: ; %Flow1
+; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 6
+; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 7
+; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:100 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
@@ -2828,15 +2850,9 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
-; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 6
-; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 7
-; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
@@ -2850,6 +2866,12 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_branch .LBB1_4
; GFX9-O0-NEXT: .LBB1_6: ; %udiv-do-while
; GFX9-O0-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: v_readlane_b32 s6, v30, 8
+; GFX9-O0-NEXT: v_readlane_b32 s7, v30, 9
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:192 ; 4-byte Folded Reload
@@ -2874,13 +2896,8 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
-; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT: v_readlane_b32 s6, v30, 8
-; GFX9-O0-NEXT: v_readlane_b32 s7, v30, 9
; GFX9-O0-NEXT: s_mov_b32 s4, 63
+; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: v_lshrrev_b64 v[28:29], s4, v[2:3]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v29
; GFX9-O0-NEXT: s_mov_b32 s5, 1
@@ -2907,6 +2924,7 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v1
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v29
+; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
; GFX9-O0-NEXT: v_mov_b32_e32 v10, v27
; GFX9-O0-NEXT: v_or3_b32 v6, v6, v7, v10
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0
@@ -2916,6 +2934,7 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v3
+; GFX9-O0-NEXT: s_waitcnt vmcnt(8)
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v25
; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v7
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2
@@ -2927,10 +2946,12 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v10, v5
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v22
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v23
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v13, v11
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v14
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v15
; GFX9-O0-NEXT: v_sub_co_u32_e32 v13, vcc, v13, v6
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_subb_co_u32_e32 v12, vcc, v12, v10, vcc
; GFX9-O0-NEXT: v_subb_co_u32_e32 v11, vcc, v11, v4, vcc
; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v7, v5, vcc
@@ -3070,6 +3091,9 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_cbranch_execnz .LBB1_6
; GFX9-O0-NEXT: s_branch .LBB1_1
; GFX9-O0-NEXT: .LBB1_7: ; %udiv-preheader
+; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
@@ -3086,12 +3110,9 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
-; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
-; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], v4, v[20:21]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-O0-NEXT: s_mov_b32 s6, 64
@@ -3172,7 +3193,6 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v14, s8
; GFX9-O0-NEXT: v_mov_b32_e32 v13, s7
; GFX9-O0-NEXT: v_mov_b32_e32 v12, s6
-; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-O0-NEXT: v_writelane_b32 v30, s4, 8
; GFX9-O0-NEXT: v_writelane_b32 v30, s5, 9
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
@@ -3204,6 +3224,9 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB1_6
; GFX9-O0-NEXT: .LBB1_8: ; %udiv-bb1
+; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
@@ -3212,12 +3235,9 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
-; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 s[6:7], 1
; GFX9-O0-NEXT: s_mov_b32 s5, s6
-; GFX9-O0-NEXT: s_waitcnt vmcnt(2)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v0
; GFX9-O0-NEXT: s_mov_b32 s4, s7
; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0
@@ -3228,7 +3248,7 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s5
; GFX9-O0-NEXT: v_add_co_u32_e32 v8, vcc, v3, v4
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v4, vcc
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s8
; GFX9-O0-NEXT: v_addc_co_u32_e32 v0, vcc, v0, v4, vcc
@@ -3332,7 +3352,6 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_mov_b64 s[6:7], exec
; GFX9-O0-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5]
; GFX9-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7]
-; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 6
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 7
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
@@ -3793,10 +3812,6 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:80 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB1_5
; GFX9-G-O0-NEXT: .LBB1_3: ; %Flow2
-; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-G-O0-NEXT: buffer_load_dword v34, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
@@ -3804,10 +3819,17 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_readlane_b32 s4, v34, 0
; GFX9-G-O0-NEXT: v_readlane_b32 s5, v34, 1
; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-G-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:136 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-G-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:140 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB1_9
; GFX9-G-O0-NEXT: .LBB1_4: ; %udiv-loop-exit
@@ -3867,6 +3889,13 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB1_3
; GFX9-G-O0-NEXT: .LBB1_5: ; %Flow1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-G-O0-NEXT: buffer_load_dword v34, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-G-O0-NEXT: v_readlane_b32 s4, v34, 4
+; GFX9-G-O0-NEXT: v_readlane_b32 s5, v34, 5
+; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
@@ -3875,17 +3904,13 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
-; GFX9-G-O0-NEXT: buffer_load_dword v34, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_readlane_b32 s4, v34, 4
-; GFX9-G-O0-NEXT: v_readlane_b32 s5, v34, 5
-; GFX9-G-O0-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:148 ; 4-byte Folded Spill
-; GFX9-G-O0-NEXT: s_nop 0
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-G-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:152 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-G-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-G-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:164 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_nop 0
@@ -3895,6 +3920,12 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_branch .LBB1_4
; GFX9-G-O0-NEXT: .LBB1_6: ; %udiv-do-while
; GFX9-G-O0-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-G-O0-NEXT: buffer_load_dword v34, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-G-O0-NEXT: v_readlane_b32 s6, v34, 6
+; GFX9-G-O0-NEXT: v_readlane_b32 s7, v34, 7
; GFX9-G-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:180 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:184 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:188 ; 4-byte Folded Reload
@@ -3919,15 +3950,11 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:248 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:252 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:256 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
-; GFX9-G-O0-NEXT: buffer_load_dword v34, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-G-O0-NEXT: v_readlane_b32 s6, v34, 6
-; GFX9-G-O0-NEXT: v_readlane_b32 s7, v34, 7
; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(18)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v2
; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v3
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v4
; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v5
; GFX9-G-O0-NEXT: s_mov_b32 s8, 1
@@ -3977,8 +4004,10 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s8
; GFX9-G-O0-NEXT: v_mov_b32_e32 v12, v0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(10)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v28, v30
; GFX9-G-O0-NEXT: v_mov_b32_e32 v29, v31
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(8)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v32
; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v33
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v28
@@ -3998,6 +4027,7 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: ; kill: def $vgpr0_vgpr1 killed $vgpr0_vgpr1 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v12
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v13
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-G-O0-NEXT: v_sub_co_u32_e64 v11, s[8:9], v11, v4
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v10, s[8:9], v10, v9, s[8:9]
; GFX9-G-O0-NEXT: v_subb_co_u32_e64 v8, s[8:9], v8, v7, s[8:9]
@@ -4118,6 +4148,9 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_cbranch_execnz .LBB1_6
; GFX9-G-O0-NEXT: s_branch .LBB1_1
; GFX9-G-O0-NEXT: .LBB1_7: ; %udiv-preheader
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-G-O0-NEXT: buffer_load_dword v34, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:260 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
@@ -4135,14 +4168,11 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
-; GFX9-G-O0-NEXT: buffer_load_dword v34, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: s_mov_b32 s4, 64
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(2)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v15, v5
; GFX9-G-O0-NEXT: v_mov_b32_e32 v14, v4
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-G-O0-NEXT: v_mov_b32_e32 v21, v7
; GFX9-G-O0-NEXT: v_mov_b32_e32 v20, v6
; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s4
@@ -4208,7 +4238,6 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], s[8:9]
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-G-O0-NEXT: v_writelane_b32 v34, s8, 6
; GFX9-G-O0-NEXT: v_writelane_b32 v34, s9, 7
; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
@@ -4240,6 +4269,9 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: s_branch .LBB1_6
; GFX9-G-O0-NEXT: .LBB1_8: ; %udiv-bb1
+; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-G-O0-NEXT: buffer_load_dword v34, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
@@ -4248,20 +4280,17 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-G-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
-; GFX9-G-O0-NEXT: buffer_load_dword v34, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-G-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-G-O0-NEXT: s_mov_b64 s[4:5], 0
; GFX9-G-O0-NEXT: s_mov_b32 s6, 1
; GFX9-G-O0-NEXT: s_mov_b32 s10, 0
; GFX9-G-O0-NEXT: s_mov_b32 s9, 0
; GFX9-G-O0-NEXT: s_mov_b32 s8, 0
; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, s6
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(4)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(3)
; GFX9-G-O0-NEXT: v_add_co_u32_e64 v4, s[6:7], v1, v4
; GFX9-G-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, s10
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(2)
+; GFX9-G-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[6:7], v3, v5, s[6:7]
; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s9
; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v7, s[6:7], v2, v3, s[6:7]
@@ -4353,7 +4382,6 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-G-O0-NEXT: s_mov_b64 s[6:7], exec
; GFX9-G-O0-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5]
; GFX9-G-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7]
-; GFX9-G-O0-NEXT: s_waitcnt vmcnt(17)
; GFX9-G-O0-NEXT: v_writelane_b32 v34, s6, 4
; GFX9-G-O0-NEXT: v_writelane_b32 v34, s7, 5
; GFX9-G-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
index f1f4abe580c002..639b2ff25dcb86 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
@@ -1600,8 +1600,14 @@ define amdgpu_kernel void @extract_neg_offset_vgpr(ptr addrspace(1) %out) {
; NOOPT-NEXT: ; implicit-def: $vgpr0
; NOOPT-NEXT: ; implicit-def: $sgpr0_sgpr1
; NOOPT-NEXT: .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; NOOPT-NEXT: s_or_saveexec_b64 s[16:17], -1
+; NOOPT-NEXT: s_waitcnt expcnt(0)
+; NOOPT-NEXT: buffer_load_dword v31, off, s[20:23], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_mov_b64 exec, s[16:17]
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: v_readlane_b32 s0, v31, 6
+; NOOPT-NEXT: v_readlane_b32 s1, v31, 7
; NOOPT-NEXT: buffer_load_dword v17, off, s[20:23], 0 offset:72 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt expcnt(1)
; NOOPT-NEXT: buffer_load_dword v0, off, s[20:23], 0 offset:4 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[20:23], 0 offset:8 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[20:23], 0 offset:12 ; 4-byte Folded Reload
@@ -1612,26 +1618,14 @@ define amdgpu_kernel void @extract_neg_offset_vgpr(ptr addrspace(1) %out) {
; NOOPT-NEXT: buffer_load_dword v7, off, s[20:23], 0 offset:32 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v8, off, s[20:23], 0 offset:36 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v9, off, s[20:23], 0 offset:40 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt expcnt(6)
; NOOPT-NEXT: buffer_load_dword v10, off, s[20:23], 0 offset:44 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt expcnt(5)
; NOOPT-NEXT: buffer_load_dword v11, off, s[20:23], 0 offset:48 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt expcnt(4)
; NOOPT-NEXT: buffer_load_dword v12, off, s[20:23], 0 offset:52 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt expcnt(3)
; NOOPT-NEXT: buffer_load_dword v13, off, s[20:23], 0 offset:56 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt expcnt(2)
; NOOPT-NEXT: buffer_load_dword v14, off, s[20:23], 0 offset:60 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt expcnt(1)
; NOOPT-NEXT: buffer_load_dword v15, off, s[20:23], 0 offset:64 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v16, off, s[20:23], 0 offset:68 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_or_saveexec_b64 s[16:17], -1
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: buffer_load_dword v31, off, s[20:23], 0 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_mov_b64 exec, s[16:17]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v31, 6
-; NOOPT-NEXT: v_readlane_b32 s1, v31, 7
; NOOPT-NEXT: v_readfirstlane_b32 s2, v16
; NOOPT-NEXT: v_cmp_eq_u32_e64 s[0:1], s2, v16
; NOOPT-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
@@ -1657,7 +1651,6 @@ define amdgpu_kernel void @extract_neg_offset_vgpr(ptr addrspace(1) %out) {
; NOOPT-NEXT: v_readlane_b32 s1, v31, 5
; NOOPT-NEXT: s_mov_b64 exec, s[0:1]
; NOOPT-NEXT: ; %bb.3:
-; NOOPT-NEXT: buffer_load_dword v0, off, s[20:23], 0 offset:76 ; 4-byte Folded Reload
; NOOPT-NEXT: s_or_saveexec_b64 s[16:17], -1
; NOOPT-NEXT: buffer_load_dword v31, off, s[20:23], 0 ; 4-byte Folded Reload
; NOOPT-NEXT: s_mov_b64 exec, s[16:17]
@@ -1666,6 +1659,8 @@ define amdgpu_kernel void @extract_neg_offset_vgpr(ptr addrspace(1) %out) {
; NOOPT-NEXT: v_readlane_b32 s1, v31, 1
; NOOPT-NEXT: v_readlane_b32 s2, v31, 2
; NOOPT-NEXT: v_readlane_b32 s3, v31, 3
+; NOOPT-NEXT: buffer_load_dword v0, off, s[20:23], 0 offset:76 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: buffer_store_dword v0, off, s[0:3], 0
; NOOPT-NEXT: s_endpgm
;
@@ -4128,6 +4123,13 @@ define amdgpu_kernel void @insert_neg_offset_vgpr(ptr addrspace(1) %in, ptr addr
; NOOPT-NEXT: buffer_store_dword v15, off, s[20:23], 0 offset:64 ; 4-byte Folded Spill
; NOOPT-NEXT: ; implicit-def: $sgpr0_sgpr1
; NOOPT-NEXT: .LBB14_1: ; =>This Inner Loop Header: Depth=1
+; NOOPT-NEXT: s_or_saveexec_b64 s[16:17], -1
+; NOOPT-NEXT: s_waitcnt expcnt(0)
+; NOOPT-NEXT: buffer_load_dword v31, off, s[20:23], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_mov_b64 exec, s[16:17]
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: v_readlane_b32 s0, v31, 6
+; NOOPT-NEXT: v_readlane_b32 s1, v31, 7
; NOOPT-NEXT: buffer_load_dword v0, off, s[20:23], 0 offset:4 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[20:23], 0 offset:8 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[20:23], 0 offset:12 ; 4-byte Folded Reload
@@ -4153,12 +4155,7 @@ define amdgpu_kernel void @insert_neg_offset_vgpr(ptr addrspace(1) %in, ptr addr
; NOOPT-NEXT: buffer_load_dword v15, off, s[20:23], 0 offset:64 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v16, off, s[20:23], 0 offset:68 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v17, off, s[20:23], 0 offset:136 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_or_saveexec_b64 s[16:17], -1
-; NOOPT-NEXT: buffer_load_dword v31, off, s[20:23], 0 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_mov_b64 exec, s[16:17]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v31, 6
-; NOOPT-NEXT: v_readlane_b32 s1, v31, 7
; NOOPT-NEXT: v_readfirstlane_b32 s2, v17
; NOOPT-NEXT: v_cmp_eq_u32_e64 s[0:1], s2, v17
; NOOPT-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
@@ -4214,6 +4211,14 @@ define amdgpu_kernel void @insert_neg_offset_vgpr(ptr addrspace(1) %in, ptr addr
; NOOPT-NEXT: v_readlane_b32 s1, v31, 5
; NOOPT-NEXT: s_mov_b64 exec, s[0:1]
; NOOPT-NEXT: ; %bb.3:
+; NOOPT-NEXT: s_or_saveexec_b64 s[16:17], -1
+; NOOPT-NEXT: buffer_load_dword v31, off, s[20:23], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_mov_b64 exec, s[16:17]
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: v_readlane_b32 s0, v31, 0
+; NOOPT-NEXT: v_readlane_b32 s1, v31, 1
+; NOOPT-NEXT: v_readlane_b32 s2, v31, 2
+; NOOPT-NEXT: v_readlane_b32 s3, v31, 3
; NOOPT-NEXT: buffer_load_dword v15, off, s[20:23], 0 offset:140 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v16, off, s[20:23], 0 offset:144 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v17, off, s[20:23], 0 offset:148 ; 4-byte Folded Reload
@@ -4230,26 +4235,22 @@ define amdgpu_kernel void @insert_neg_offset_vgpr(ptr addrspace(1) %in, ptr addr
; NOOPT-NEXT: buffer_load_dword v28, off, s[20:23], 0 offset:192 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v29, off, s[20:23], 0 offset:196 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v30, off, s[20:23], 0 offset:200 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_or_saveexec_b64 s[16:17], -1
-; NOOPT-NEXT: buffer_load_dword v31, off, s[20:23], 0 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_mov_b64 exec, s[16:17]
-; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v31, 0
-; NOOPT-NEXT: v_readlane_b32 s1, v31, 1
-; NOOPT-NEXT: v_readlane_b32 s2, v31, 2
-; NOOPT-NEXT: v_readlane_b32 s3, v31, 3
+; NOOPT-NEXT: s_waitcnt vmcnt(12)
; NOOPT-NEXT: v_mov_b32_e32 v4, v18
; NOOPT-NEXT: v_mov_b32_e32 v5, v17
; NOOPT-NEXT: v_mov_b32_e32 v6, v16
; NOOPT-NEXT: v_mov_b32_e32 v0, v15
+; NOOPT-NEXT: s_waitcnt vmcnt(8)
; NOOPT-NEXT: v_mov_b32_e32 v1, v22
; NOOPT-NEXT: v_mov_b32_e32 v2, v21
; NOOPT-NEXT: v_mov_b32_e32 v3, v20
; NOOPT-NEXT: v_mov_b32_e32 v7, v19
+; NOOPT-NEXT: s_waitcnt vmcnt(4)
; NOOPT-NEXT: v_mov_b32_e32 v12, v26
; NOOPT-NEXT: v_mov_b32_e32 v13, v25
; NOOPT-NEXT: v_mov_b32_e32 v14, v24
; NOOPT-NEXT: v_mov_b32_e32 v8, v23
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_mov_b32_e32 v9, v30
; NOOPT-NEXT: v_mov_b32_e32 v10, v29
; NOOPT-NEXT: v_mov_b32_e32 v11, v28
@@ -4611,6 +4612,13 @@ define amdgpu_kernel void @insert_neg_inline_offset_vgpr(ptr addrspace(1) %in, p
; NOOPT-NEXT: buffer_store_dword v15, off, s[20:23], 0 offset:64 ; 4-byte Folded Spill
; NOOPT-NEXT: ; implicit-def: $sgpr0_sgpr1
; NOOPT-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; NOOPT-NEXT: s_or_saveexec_b64 s[16:17], -1
+; NOOPT-NEXT: s_waitcnt expcnt(0)
+; NOOPT-NEXT: buffer_load_dword v31, off, s[20:23], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_mov_b64 exec, s[16:17]
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: v_readlane_b32 s0, v31, 6
+; NOOPT-NEXT: v_readlane_b32 s1, v31, 7
; NOOPT-NEXT: buffer_load_dword v0, off, s[20:23], 0 offset:4 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[20:23], 0 offset:8 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[20:23], 0 offset:12 ; 4-byte Folded Reload
@@ -4636,12 +4644,7 @@ define amdgpu_kernel void @insert_neg_inline_offset_vgpr(ptr addrspace(1) %in, p
; NOOPT-NEXT: buffer_load_dword v15, off, s[20:23], 0 offset:64 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v16, off, s[20:23], 0 offset:68 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v17, off, s[20:23], 0 offset:136 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_or_saveexec_b64 s[16:17], -1
-; NOOPT-NEXT: buffer_load_dword v31, off, s[20:23], 0 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_mov_b64 exec, s[16:17]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v31, 6
-; NOOPT-NEXT: v_readlane_b32 s1, v31, 7
; NOOPT-NEXT: v_readfirstlane_b32 s2, v17
; NOOPT-NEXT: v_cmp_eq_u32_e64 s[0:1], s2, v17
; NOOPT-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
@@ -4697,6 +4700,14 @@ define amdgpu_kernel void @insert_neg_inline_offset_vgpr(ptr addrspace(1) %in, p
; NOOPT-NEXT: v_readlane_b32 s1, v31, 5
; NOOPT-NEXT: s_mov_b64 exec, s[0:1]
; NOOPT-NEXT: ; %bb.3:
+; NOOPT-NEXT: s_or_saveexec_b64 s[16:17], -1
+; NOOPT-NEXT: buffer_load_dword v31, off, s[20:23], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_mov_b64 exec, s[16:17]
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: v_readlane_b32 s0, v31, 0
+; NOOPT-NEXT: v_readlane_b32 s1, v31, 1
+; NOOPT-NEXT: v_readlane_b32 s2, v31, 2
+; NOOPT-NEXT: v_readlane_b32 s3, v31, 3
; NOOPT-NEXT: buffer_load_dword v15, off, s[20:23], 0 offset:140 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v16, off, s[20:23], 0 offset:144 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v17, off, s[20:23], 0 offset:148 ; 4-byte Folded Reload
@@ -4713,26 +4724,22 @@ define amdgpu_kernel void @insert_neg_inline_offset_vgpr(ptr addrspace(1) %in, p
; NOOPT-NEXT: buffer_load_dword v28, off, s[20:23], 0 offset:192 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v29, off, s[20:23], 0 offset:196 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v30, off, s[20:23], 0 offset:200 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_or_saveexec_b64 s[16:17], -1
-; NOOPT-NEXT: buffer_load_dword v31, off, s[20:23], 0 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_mov_b64 exec, s[16:17]
-; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v31, 0
-; NOOPT-NEXT: v_readlane_b32 s1, v31, 1
-; NOOPT-NEXT: v_readlane_b32 s2, v31, 2
-; NOOPT-NEXT: v_readlane_b32 s3, v31, 3
+; NOOPT-NEXT: s_waitcnt vmcnt(12)
; NOOPT-NEXT: v_mov_b32_e32 v4, v18
; NOOPT-NEXT: v_mov_b32_e32 v5, v17
; NOOPT-NEXT: v_mov_b32_e32 v6, v16
; NOOPT-NEXT: v_mov_b32_e32 v0, v15
+; NOOPT-NEXT: s_waitcnt vmcnt(8)
; NOOPT-NEXT: v_mov_b32_e32 v1, v22
; NOOPT-NEXT: v_mov_b32_e32 v2, v21
; NOOPT-NEXT: v_mov_b32_e32 v3, v20
; NOOPT-NEXT: v_mov_b32_e32 v7, v19
+; NOOPT-NEXT: s_waitcnt vmcnt(4)
; NOOPT-NEXT: v_mov_b32_e32 v12, v26
; NOOPT-NEXT: v_mov_b32_e32 v13, v25
; NOOPT-NEXT: v_mov_b32_e32 v14, v24
; NOOPT-NEXT: v_mov_b32_e32 v8, v23
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_mov_b32_e32 v9, v30
; NOOPT-NEXT: v_mov_b32_e32 v10, v29
; NOOPT-NEXT: v_mov_b32_e32 v11, v28
@@ -5163,8 +5170,14 @@ define amdgpu_kernel void @extract_vgpr_offset_multiple_in_block(ptr addrspace(1
; NOOPT-NEXT: ; implicit-def: $vgpr0
; NOOPT-NEXT: ; implicit-def: $sgpr0_sgpr1
; NOOPT-NEXT: .LBB16_1: ; =>This Inner Loop Header: Depth=1
+; NOOPT-NEXT: s_or_saveexec_b64 s[28:29], -1
+; NOOPT-NEXT: s_waitcnt expcnt(0)
+; NOOPT-NEXT: buffer_load_dword v18, off, s[36:39], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_mov_b64 exec, s[28:29]
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: v_readlane_b32 s0, v18, 23
+; NOOPT-NEXT: v_readlane_b32 s1, v18, 24
; NOOPT-NEXT: buffer_load_dword v17, off, s[36:39], 0 offset:80 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt expcnt(1)
; NOOPT-NEXT: buffer_load_dword v0, off, s[36:39], 0 offset:4 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[36:39], 0 offset:8 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[36:39], 0 offset:12 ; 4-byte Folded Reload
@@ -5175,26 +5188,14 @@ define amdgpu_kernel void @extract_vgpr_offset_multiple_in_block(ptr addrspace(1
; NOOPT-NEXT: buffer_load_dword v7, off, s[36:39], 0 offset:32 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v8, off, s[36:39], 0 offset:36 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v9, off, s[36:39], 0 offset:40 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt expcnt(6)
; NOOPT-NEXT: buffer_load_dword v10, off, s[36:39], 0 offset:44 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt expcnt(5)
; NOOPT-NEXT: buffer_load_dword v11, off, s[36:39], 0 offset:48 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt expcnt(4)
; NOOPT-NEXT: buffer_load_dword v12, off, s[36:39], 0 offset:52 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt expcnt(3)
; NOOPT-NEXT: buffer_load_dword v13, off, s[36:39], 0 offset:56 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt expcnt(2)
; NOOPT-NEXT: buffer_load_dword v14, off, s[36:39], 0 offset:60 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt expcnt(1)
; NOOPT-NEXT: buffer_load_dword v15, off, s[36:39], 0 offset:64 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v16, off, s[36:39], 0 offset:72 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_or_saveexec_b64 s[28:29], -1
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: buffer_load_dword v18, off, s[36:39], 0 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_mov_b64 exec, s[28:29]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v18, 23
-; NOOPT-NEXT: v_readlane_b32 s1, v18, 24
; NOOPT-NEXT: v_readfirstlane_b32 s2, v16
; NOOPT-NEXT: v_cmp_eq_u32_e64 s[0:1], s2, v16
; NOOPT-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
@@ -5286,8 +5287,14 @@ define amdgpu_kernel void @extract_vgpr_offset_multiple_in_block(ptr addrspace(1
; NOOPT-NEXT: ; implicit-def: $vgpr0
; NOOPT-NEXT: ; implicit-def: $sgpr0_sgpr1
; NOOPT-NEXT: .LBB16_4: ; =>This Inner Loop Header: Depth=1
+; NOOPT-NEXT: s_or_saveexec_b64 s[28:29], -1
+; NOOPT-NEXT: s_waitcnt expcnt(0)
+; NOOPT-NEXT: buffer_load_dword v18, off, s[36:39], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_mov_b64 exec, s[28:29]
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: v_readlane_b32 s0, v18, 28
+; NOOPT-NEXT: v_readlane_b32 s1, v18, 29
; NOOPT-NEXT: buffer_load_dword v17, off, s[36:39], 0 offset:152 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt expcnt(1)
; NOOPT-NEXT: buffer_load_dword v0, off, s[36:39], 0 offset:88 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[36:39], 0 offset:92 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[36:39], 0 offset:96 ; 4-byte Folded Reload
@@ -5298,26 +5305,14 @@ define amdgpu_kernel void @extract_vgpr_offset_multiple_in_block(ptr addrspace(1
; NOOPT-NEXT: buffer_load_dword v7, off, s[36:39], 0 offset:116 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v8, off, s[36:39], 0 offset:120 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v9, off, s[36:39], 0 offset:124 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt expcnt(6)
; NOOPT-NEXT: buffer_load_dword v10, off, s[36:39], 0 offset:128 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt expcnt(5)
; NOOPT-NEXT: buffer_load_dword v11, off, s[36:39], 0 offset:132 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt expcnt(4)
; NOOPT-NEXT: buffer_load_dword v12, off, s[36:39], 0 offset:136 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt expcnt(3)
; NOOPT-NEXT: buffer_load_dword v13, off, s[36:39], 0 offset:140 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt expcnt(2)
; NOOPT-NEXT: buffer_load_dword v14, off, s[36:39], 0 offset:144 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_waitcnt expcnt(1)
; NOOPT-NEXT: buffer_load_dword v15, off, s[36:39], 0 offset:148 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v16, off, s[36:39], 0 offset:68 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_or_saveexec_b64 s[28:29], -1
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: buffer_load_dword v18, off, s[36:39], 0 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_mov_b64 exec, s[28:29]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v18, 28
-; NOOPT-NEXT: v_readlane_b32 s1, v18, 29
; NOOPT-NEXT: v_readfirstlane_b32 s2, v16
; NOOPT-NEXT: v_cmp_eq_u32_e64 s[0:1], s2, v16
; NOOPT-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
@@ -5343,9 +5338,6 @@ define amdgpu_kernel void @extract_vgpr_offset_multiple_in_block(ptr addrspace(1
; NOOPT-NEXT: v_readlane_b32 s1, v18, 27
; NOOPT-NEXT: s_mov_b64 exec, s[0:1]
; NOOPT-NEXT: ; %bb.6:
-; NOOPT-NEXT: buffer_load_dword v0, off, s[36:39], 0 offset:76 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v1, off, s[36:39], 0 offset:156 ; 4-byte Folded Reload
-; NOOPT-NEXT: buffer_load_dword v2, off, s[36:39], 0 offset:84 ; 4-byte Folded Reload
; NOOPT-NEXT: s_or_saveexec_b64 s[28:29], -1
; NOOPT-NEXT: buffer_load_dword v18, off, s[36:39], 0 ; 4-byte Folded Reload
; NOOPT-NEXT: s_mov_b64 exec, s[28:29]
@@ -5355,6 +5347,10 @@ define amdgpu_kernel void @extract_vgpr_offset_multiple_in_block(ptr addrspace(1
; NOOPT-NEXT: v_readlane_b32 s5, v18, 1
; NOOPT-NEXT: v_readlane_b32 s6, v18, 2
; NOOPT-NEXT: v_readlane_b32 s7, v18, 3
+; NOOPT-NEXT: buffer_load_dword v0, off, s[36:39], 0 offset:76 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v1, off, s[36:39], 0 offset:156 ; 4-byte Folded Reload
+; NOOPT-NEXT: buffer_load_dword v2, off, s[36:39], 0 offset:84 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: buffer_store_dword v2, off, s[4:7], 0
; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: buffer_store_dword v1, off, s[4:7], 0
@@ -5903,6 +5899,13 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: buffer_store_dword v15, off, s[28:31], 0 offset:64 ; 4-byte Folded Spill
; NOOPT-NEXT: ; implicit-def: $sgpr0_sgpr1
; NOOPT-NEXT: .LBB17_1: ; =>This Inner Loop Header: Depth=1
+; NOOPT-NEXT: s_or_saveexec_b64 s[26:27], -1
+; NOOPT-NEXT: s_waitcnt expcnt(0)
+; NOOPT-NEXT: buffer_load_dword v32, off, s[28:31], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_mov_b64 exec, s[26:27]
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: v_readlane_b32 s0, v32, 7
+; NOOPT-NEXT: v_readlane_b32 s1, v32, 8
; NOOPT-NEXT: buffer_load_dword v0, off, s[28:31], 0 offset:4 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[28:31], 0 offset:8 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[28:31], 0 offset:12 ; 4-byte Folded Reload
@@ -5928,12 +5931,7 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: buffer_load_dword v15, off, s[28:31], 0 offset:64 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v16, off, s[28:31], 0 offset:72 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v17, off, s[28:31], 0 offset:80 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_or_saveexec_b64 s[26:27], -1
-; NOOPT-NEXT: buffer_load_dword v32, off, s[28:31], 0 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_mov_b64 exec, s[26:27]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v32, 7
-; NOOPT-NEXT: v_readlane_b32 s1, v32, 8
; NOOPT-NEXT: v_readfirstlane_b32 s2, v17
; NOOPT-NEXT: v_cmp_eq_u32_e64 s[0:1], s2, v17
; NOOPT-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
@@ -5989,6 +5987,9 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: v_readlane_b32 s1, v32, 6
; NOOPT-NEXT: s_mov_b64 exec, s[0:1]
; NOOPT-NEXT: ; %bb.3:
+; NOOPT-NEXT: s_or_saveexec_b64 s[26:27], -1
+; NOOPT-NEXT: buffer_load_dword v32, off, s[28:31], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_mov_b64 exec, s[26:27]
; NOOPT-NEXT: buffer_load_dword v0, off, s[28:31], 0 offset:88 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[28:31], 0 offset:92 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[28:31], 0 offset:96 ; 4-byte Folded Reload
@@ -6005,13 +6006,10 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: buffer_load_dword v13, off, s[28:31], 0 offset:140 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v14, off, s[28:31], 0 offset:144 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v15, off, s[28:31], 0 offset:148 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_or_saveexec_b64 s[26:27], -1
-; NOOPT-NEXT: buffer_load_dword v32, off, s[28:31], 0 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_mov_b64 exec, s[26:27]
; NOOPT-NEXT: v_mov_b32_e32 v16, 63
; NOOPT-NEXT: buffer_store_dword v16, off, s[28:31], 0 offset:216 ; 4-byte Folded Spill
; NOOPT-NEXT: s_mov_b64 s[0:1], exec
-; NOOPT-NEXT: s_waitcnt vmcnt(1)
+; NOOPT-NEXT: s_waitcnt vmcnt(14)
; NOOPT-NEXT: v_writelane_b32 v32, s0, 9
; NOOPT-NEXT: v_writelane_b32 v32, s1, 10
; NOOPT-NEXT: s_or_saveexec_b64 s[26:27], -1
@@ -6020,21 +6018,32 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: buffer_store_dword v0, off, s[28:31], 0 offset:152 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v1, off, s[28:31], 0 offset:156 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v2, off, s[28:31], 0 offset:160 ; 4-byte Folded Spill
+; NOOPT-NEXT: s_waitcnt vmcnt(14)
; NOOPT-NEXT: buffer_store_dword v3, off, s[28:31], 0 offset:164 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v4, off, s[28:31], 0 offset:168 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v5, off, s[28:31], 0 offset:172 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v6, off, s[28:31], 0 offset:176 ; 4-byte Folded Spill
+; NOOPT-NEXT: s_waitcnt vmcnt(14)
; NOOPT-NEXT: buffer_store_dword v7, off, s[28:31], 0 offset:180 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v8, off, s[28:31], 0 offset:184 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v9, off, s[28:31], 0 offset:188 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v10, off, s[28:31], 0 offset:192 ; 4-byte Folded Spill
+; NOOPT-NEXT: s_waitcnt vmcnt(14)
; NOOPT-NEXT: buffer_store_dword v11, off, s[28:31], 0 offset:196 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v12, off, s[28:31], 0 offset:200 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v13, off, s[28:31], 0 offset:204 ; 4-byte Folded Spill
; NOOPT-NEXT: buffer_store_dword v14, off, s[28:31], 0 offset:208 ; 4-byte Folded Spill
+; NOOPT-NEXT: s_waitcnt vmcnt(14)
; NOOPT-NEXT: buffer_store_dword v15, off, s[28:31], 0 offset:212 ; 4-byte Folded Spill
; NOOPT-NEXT: ; implicit-def: $sgpr0_sgpr1
; NOOPT-NEXT: .LBB17_4: ; =>This Inner Loop Header: Depth=1
+; NOOPT-NEXT: s_or_saveexec_b64 s[26:27], -1
+; NOOPT-NEXT: s_waitcnt expcnt(0)
+; NOOPT-NEXT: buffer_load_dword v32, off, s[28:31], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_mov_b64 exec, s[26:27]
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: v_readlane_b32 s0, v32, 11
+; NOOPT-NEXT: v_readlane_b32 s1, v32, 12
; NOOPT-NEXT: buffer_load_dword v0, off, s[28:31], 0 offset:152 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[28:31], 0 offset:156 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[28:31], 0 offset:160 ; 4-byte Folded Reload
@@ -6060,12 +6069,7 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: buffer_load_dword v15, off, s[28:31], 0 offset:212 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v16, off, s[28:31], 0 offset:216 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v17, off, s[28:31], 0 offset:76 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_or_saveexec_b64 s[26:27], -1
-; NOOPT-NEXT: buffer_load_dword v32, off, s[28:31], 0 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_mov_b64 exec, s[26:27]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v32, 11
-; NOOPT-NEXT: v_readlane_b32 s1, v32, 12
; NOOPT-NEXT: v_readfirstlane_b32 s2, v17
; NOOPT-NEXT: v_cmp_eq_u32_e64 s[0:1], s2, v17
; NOOPT-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
@@ -6121,6 +6125,15 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: v_readlane_b32 s1, v32, 10
; NOOPT-NEXT: s_mov_b64 exec, s[0:1]
; NOOPT-NEXT: ; %bb.6:
+; NOOPT-NEXT: s_or_saveexec_b64 s[26:27], -1
+; NOOPT-NEXT: buffer_load_dword v32, off, s[28:31], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_mov_b64 exec, s[26:27]
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: v_readlane_b32 s0, v32, 4
+; NOOPT-NEXT: v_readlane_b32 s4, v32, 0
+; NOOPT-NEXT: v_readlane_b32 s5, v32, 1
+; NOOPT-NEXT: v_readlane_b32 s6, v32, 2
+; NOOPT-NEXT: v_readlane_b32 s7, v32, 3
; NOOPT-NEXT: buffer_load_dword v0, off, s[28:31], 0 offset:84 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v16, off, s[28:31], 0 offset:220 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v17, off, s[28:31], 0 offset:224 ; 4-byte Folded Reload
@@ -6138,27 +6151,22 @@ define amdgpu_kernel void @insert_vgpr_offset_multiple_in_block(ptr addrspace(1)
; NOOPT-NEXT: buffer_load_dword v29, off, s[28:31], 0 offset:272 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v30, off, s[28:31], 0 offset:276 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v31, off, s[28:31], 0 offset:280 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_or_saveexec_b64 s[26:27], -1
-; NOOPT-NEXT: buffer_load_dword v32, off, s[28:31], 0 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_mov_b64 exec, s[26:27]
-; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v32, 4
-; NOOPT-NEXT: v_readlane_b32 s4, v32, 0
-; NOOPT-NEXT: v_readlane_b32 s5, v32, 1
-; NOOPT-NEXT: v_readlane_b32 s6, v32, 2
-; NOOPT-NEXT: v_readlane_b32 s7, v32, 3
+; NOOPT-NEXT: s_waitcnt vmcnt(12)
; NOOPT-NEXT: v_mov_b32_e32 v5, v19
; NOOPT-NEXT: v_mov_b32_e32 v6, v18
; NOOPT-NEXT: v_mov_b32_e32 v7, v17
; NOOPT-NEXT: v_mov_b32_e32 v1, v16
+; NOOPT-NEXT: s_waitcnt vmcnt(8)
; NOOPT-NEXT: v_mov_b32_e32 v2, v23
; NOOPT-NEXT: v_mov_b32_e32 v3, v22
; NOOPT-NEXT: v_mov_b32_e32 v4, v21
; NOOPT-NEXT: v_mov_b32_e32 v8, v20
+; NOOPT-NEXT: s_waitcnt vmcnt(4)
; NOOPT-NEXT: v_mov_b32_e32 v13, v27
; NOOPT-NEXT: v_mov_b32_e32 v14, v26
; NOOPT-NEXT: v_mov_b32_e32 v15, v25
; NOOPT-NEXT: v_mov_b32_e32 v9, v24
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_mov_b32_e32 v10, v31
; NOOPT-NEXT: v_mov_b32_e32 v11, v30
; NOOPT-NEXT: v_mov_b32_e32 v12, v29
@@ -9071,15 +9079,17 @@ define amdgpu_kernel void @broken_phi_bb(i32 %arg, i32 %arg1) {
; NOOPT-NEXT: .LBB26_1: ; %bb2
; NOOPT-NEXT: ; =>This Loop Header: Depth=1
; NOOPT-NEXT: ; Child Loop BB26_3 Depth 2
-; NOOPT-NEXT: s_waitcnt expcnt(0)
-; NOOPT-NEXT: buffer_load_dword v0, off, s[24:27], 0 offset:4 ; 4-byte Folded Reload
; NOOPT-NEXT: s_or_saveexec_b64 s[20:21], -1
+; NOOPT-NEXT: s_waitcnt expcnt(1)
; NOOPT-NEXT: buffer_load_dword v18, off, s[24:27], 0 ; 4-byte Folded Reload
; NOOPT-NEXT: s_mov_b64 exec, s[20:21]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_readlane_b32 s2, v18, 0
+; NOOPT-NEXT: s_waitcnt expcnt(0)
+; NOOPT-NEXT: buffer_load_dword v0, off, s[24:27], 0 offset:4 ; 4-byte Folded Reload
; NOOPT-NEXT: s_mov_b64 s[0:1], -1
; NOOPT-NEXT: ; implicit-def: $sgpr4
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_cmp_ge_i32_e64 s[2:3], v0, s2
; NOOPT-NEXT: v_mov_b32_e32 v0, s4
; NOOPT-NEXT: s_and_b64 vcc, exec, s[2:3]
@@ -9156,6 +9166,13 @@ define amdgpu_kernel void @broken_phi_bb(i32 %arg, i32 %arg1) {
; NOOPT-NEXT: ; implicit-def: $sgpr0_sgpr1
; NOOPT-NEXT: .LBB26_3: ; Parent Loop BB26_1 Depth=1
; NOOPT-NEXT: ; => This Inner Loop Header: Depth=2
+; NOOPT-NEXT: s_or_saveexec_b64 s[20:21], -1
+; NOOPT-NEXT: s_waitcnt expcnt(0)
+; NOOPT-NEXT: buffer_load_dword v18, off, s[24:27], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_mov_b64 exec, s[20:21]
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: v_readlane_b32 s0, v18, 6
+; NOOPT-NEXT: v_readlane_b32 s1, v18, 7
; NOOPT-NEXT: buffer_load_dword v0, off, s[24:27], 0 offset:12 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[24:27], 0 offset:16 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[24:27], 0 offset:20 ; 4-byte Folded Reload
@@ -9181,12 +9198,7 @@ define amdgpu_kernel void @broken_phi_bb(i32 %arg, i32 %arg1) {
; NOOPT-NEXT: buffer_load_dword v15, off, s[24:27], 0 offset:72 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v16, off, s[24:27], 0 offset:76 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v17, off, s[24:27], 0 offset:80 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_or_saveexec_b64 s[20:21], -1
-; NOOPT-NEXT: buffer_load_dword v18, off, s[24:27], 0 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_mov_b64 exec, s[20:21]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v18, 6
-; NOOPT-NEXT: v_readlane_b32 s1, v18, 7
; NOOPT-NEXT: v_readfirstlane_b32 s2, v17
; NOOPT-NEXT: v_cmp_eq_u32_e64 s[0:1], s2, v17
; NOOPT-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
@@ -9242,6 +9254,9 @@ define amdgpu_kernel void @broken_phi_bb(i32 %arg, i32 %arg1) {
; NOOPT-NEXT: v_readlane_b32 s1, v18, 5
; NOOPT-NEXT: s_mov_b64 exec, s[0:1]
; NOOPT-NEXT: ; %bb.5: ; in Loop: Header=BB26_1 Depth=1
+; NOOPT-NEXT: s_or_saveexec_b64 s[20:21], -1
+; NOOPT-NEXT: buffer_load_dword v18, off, s[24:27], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_mov_b64 exec, s[20:21]
; NOOPT-NEXT: buffer_load_dword v0, off, s[24:27], 0 offset:84 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[24:27], 0 offset:88 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[24:27], 0 offset:92 ; 4-byte Folded Reload
@@ -9258,13 +9273,9 @@ define amdgpu_kernel void @broken_phi_bb(i32 %arg, i32 %arg1) {
; NOOPT-NEXT: buffer_load_dword v13, off, s[24:27], 0 offset:136 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v14, off, s[24:27], 0 offset:140 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v15, off, s[24:27], 0 offset:144 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_or_saveexec_b64 s[20:21], -1
-; NOOPT-NEXT: buffer_load_dword v18, off, s[24:27], 0 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_mov_b64 exec, s[20:21]
; NOOPT-NEXT: s_mov_b64 s[0:1], 0
; NOOPT-NEXT: s_waitcnt vmcnt(14)
; NOOPT-NEXT: buffer_store_dword v0, off, s[24:27], 0 offset:8 ; 4-byte Folded Spill
-; NOOPT-NEXT: s_waitcnt vmcnt(1)
; NOOPT-NEXT: v_writelane_b32 v18, s0, 2
; NOOPT-NEXT: v_writelane_b32 v18, s1, 3
; NOOPT-NEXT: s_or_saveexec_b64 s[20:21], -1
@@ -9272,8 +9283,6 @@ define amdgpu_kernel void @broken_phi_bb(i32 %arg, i32 %arg1) {
; NOOPT-NEXT: s_mov_b64 exec, s[20:21]
; NOOPT-NEXT: .LBB26_6: ; %Flow
; NOOPT-NEXT: ; in Loop: Header=BB26_1 Depth=1
-; NOOPT-NEXT: s_waitcnt expcnt(1)
-; NOOPT-NEXT: buffer_load_dword v0, off, s[24:27], 0 offset:8 ; 4-byte Folded Reload
; NOOPT-NEXT: s_or_saveexec_b64 s[20:21], -1
; NOOPT-NEXT: s_waitcnt expcnt(0)
; NOOPT-NEXT: buffer_load_dword v18, off, s[24:27], 0 ; 4-byte Folded Reload
@@ -9281,11 +9290,13 @@ define amdgpu_kernel void @broken_phi_bb(i32 %arg, i32 %arg1) {
; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_readlane_b32 s0, v18, 2
; NOOPT-NEXT: v_readlane_b32 s1, v18, 3
+; NOOPT-NEXT: buffer_load_dword v0, off, s[24:27], 0 offset:8 ; 4-byte Folded Reload
; NOOPT-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[0:1]
; NOOPT-NEXT: s_mov_b32 s0, 1
; NOOPT-NEXT: ; implicit-def: $sgpr1
; NOOPT-NEXT: v_cmp_ne_u32_e64 s[0:1], v1, s0
; NOOPT-NEXT: s_and_b64 vcc, exec, s[0:1]
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: buffer_store_dword v0, off, s[24:27], 0 offset:4 ; 4-byte Folded Spill
; NOOPT-NEXT: s_cbranch_vccnz .LBB26_1
; NOOPT-NEXT: ; %bb.7: ; %bb8
@@ -9645,6 +9656,13 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; NOOPT-NEXT: buffer_store_dword v15, off, s[16:19], 0 offset:64 ; 4-byte Folded Spill
; NOOPT-NEXT: ; implicit-def: $sgpr0_sgpr1
; NOOPT-NEXT: .LBB27_1: ; =>This Inner Loop Header: Depth=1
+; NOOPT-NEXT: s_or_saveexec_b64 s[12:13], -1
+; NOOPT-NEXT: s_waitcnt expcnt(0)
+; NOOPT-NEXT: buffer_load_dword v33, off, s[16:19], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_mov_b64 exec, s[12:13]
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: v_readlane_b32 s0, v33, 9
+; NOOPT-NEXT: v_readlane_b32 s1, v33, 10
; NOOPT-NEXT: buffer_load_dword v0, off, s[16:19], 0 offset:4 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v1, off, s[16:19], 0 offset:8 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v2, off, s[16:19], 0 offset:12 ; 4-byte Folded Reload
@@ -9670,12 +9688,7 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; NOOPT-NEXT: buffer_load_dword v15, off, s[16:19], 0 offset:64 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v16, off, s[16:19], 0 offset:144 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v17, off, s[16:19], 0 offset:132 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_or_saveexec_b64 s[12:13], -1
-; NOOPT-NEXT: buffer_load_dword v33, off, s[16:19], 0 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_mov_b64 exec, s[12:13]
; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v33, 9
-; NOOPT-NEXT: v_readlane_b32 s1, v33, 10
; NOOPT-NEXT: v_readfirstlane_b32 s2, v17
; NOOPT-NEXT: v_cmp_eq_u32_e64 s[0:1], s2, v17
; NOOPT-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
@@ -9731,6 +9744,14 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; NOOPT-NEXT: v_readlane_b32 s1, v33, 8
; NOOPT-NEXT: s_mov_b64 exec, s[0:1]
; NOOPT-NEXT: ; %bb.3:
+; NOOPT-NEXT: s_or_saveexec_b64 s[12:13], -1
+; NOOPT-NEXT: buffer_load_dword v33, off, s[16:19], 0 ; 4-byte Folded Reload
+; NOOPT-NEXT: s_mov_b64 exec, s[12:13]
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
+; NOOPT-NEXT: v_readlane_b32 s0, v33, 3
+; NOOPT-NEXT: v_readlane_b32 s1, v33, 4
+; NOOPT-NEXT: v_readlane_b32 s2, v33, 5
+; NOOPT-NEXT: v_readlane_b32 s3, v33, 6
; NOOPT-NEXT: buffer_load_dword v4, off, s[16:19], 0 offset:136 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v5, off, s[16:19], 0 offset:140 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v17, off, s[16:19], 0 offset:148 ; 4-byte Folded Reload
@@ -9749,26 +9770,22 @@ define amdgpu_cs void @insert_or_disj_index(ptr addrspace(1) %out, ptr addrspace
; NOOPT-NEXT: buffer_load_dword v30, off, s[16:19], 0 offset:200 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v31, off, s[16:19], 0 offset:204 ; 4-byte Folded Reload
; NOOPT-NEXT: buffer_load_dword v32, off, s[16:19], 0 offset:208 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_or_saveexec_b64 s[12:13], -1
-; NOOPT-NEXT: buffer_load_dword v33, off, s[16:19], 0 ; 4-byte Folded Reload
-; NOOPT-NEXT: s_mov_b64 exec, s[12:13]
-; NOOPT-NEXT: s_waitcnt vmcnt(0)
-; NOOPT-NEXT: v_readlane_b32 s0, v33, 3
-; NOOPT-NEXT: v_readlane_b32 s1, v33, 4
-; NOOPT-NEXT: v_readlane_b32 s2, v33, 5
-; NOOPT-NEXT: v_readlane_b32 s3, v33, 6
+; NOOPT-NEXT: s_waitcnt vmcnt(12)
; NOOPT-NEXT: v_mov_b32_e32 v6, v20
; NOOPT-NEXT: v_mov_b32_e32 v7, v19
; NOOPT-NEXT: v_mov_b32_e32 v8, v18
; NOOPT-NEXT: v_mov_b32_e32 v0, v17
+; NOOPT-NEXT: s_waitcnt vmcnt(8)
; NOOPT-NEXT: v_mov_b32_e32 v1, v24
; NOOPT-NEXT: v_mov_b32_e32 v2, v23
; NOOPT-NEXT: v_mov_b32_e32 v3, v22
; NOOPT-NEXT: v_mov_b32_e32 v9, v21
+; NOOPT-NEXT: s_waitcnt vmcnt(4)
; NOOPT-NEXT: v_mov_b32_e32 v14, v28
; NOOPT-NEXT: v_mov_b32_e32 v15, v27
; NOOPT-NEXT: v_mov_b32_e32 v16, v26
; NOOPT-NEXT: v_mov_b32_e32 v10, v25
+; NOOPT-NEXT: s_waitcnt vmcnt(0)
; NOOPT-NEXT: v_mov_b32_e32 v11, v32
; NOOPT-NEXT: v_mov_b32_e32 v12, v31
; NOOPT-NEXT: v_mov_b32_e32 v13, v30
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
index 76a31a7fac8c1a..e44803d611f84b 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands-non-ptr-intrinsics.ll
@@ -171,23 +171,23 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; W64-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
+; W64-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
-; W64-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
-; W64-O0-NEXT: s_waitcnt vmcnt(4)
-; W64-O0-NEXT: v_readfirstlane_b32 s8, v0
; W64-O0-NEXT: s_waitcnt vmcnt(3)
+; W64-O0-NEXT: v_readfirstlane_b32 s8, v0
+; W64-O0-NEXT: s_waitcnt vmcnt(2)
; W64-O0-NEXT: v_readfirstlane_b32 s12, v1
; W64-O0-NEXT: s_mov_b32 s4, s8
; W64-O0-NEXT: s_mov_b32 s5, s12
; W64-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], s[4:5], v[0:1]
-; W64-O0-NEXT: s_waitcnt vmcnt(2)
-; W64-O0-NEXT: v_readfirstlane_b32 s7, v2
; W64-O0-NEXT: s_waitcnt vmcnt(1)
+; W64-O0-NEXT: v_readfirstlane_b32 s7, v2
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_readfirstlane_b32 s6, v3
; W64-O0-NEXT: s_mov_b32 s10, s7
; W64-O0-NEXT: s_mov_b32 s11, s6
@@ -197,7 +197,6 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; W64-O0-NEXT: s_mov_b32 s9, s12
; W64-O0-NEXT: s_mov_b32 s10, s7
; W64-O0-NEXT: s_mov_b32 s11, s6
-; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_writelane_b32 v7, s8, 3
; W64-O0-NEXT: v_writelane_b32 v7, s9, 4
; W64-O0-NEXT: v_writelane_b32 v7, s10, 5
@@ -209,7 +208,6 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; W64-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: ; %bb.2: ; in Loop: Header=BB0_1 Depth=1
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
; W64-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
@@ -221,14 +219,15 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; W64-O0-NEXT: v_readlane_b32 s10, v7, 5
; W64-O0-NEXT: v_readlane_b32 s11, v7, 6
; W64-O0-NEXT: v_readlane_b32 s6, v7, 0
-; W64-O0-NEXT: s_nop 4
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
+; W64-O0-NEXT: s_nop 2
; W64-O0-NEXT: buffer_load_format_x v0, v0, s[8:11], s6 idxen
; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
; W64-O0-NEXT: s_xor_b64 exec, exec, s[4:5]
; W64-O0-NEXT: s_cbranch_execnz .LBB0_1
; W64-O0-NEXT: ; %bb.3:
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
; W64-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
@@ -236,6 +235,7 @@ define float @mubuf_vgpr(<4 x i32> %i, i32 %c) #0 {
; W64-O0-NEXT: v_readlane_b32 s4, v7, 1
; W64-O0-NEXT: v_readlane_b32 s5, v7, 2
; W64-O0-NEXT: s_mov_b64 exec, s[4:5]
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; W64-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1
; W64-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[4:5]
@@ -563,23 +563,23 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; W64-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
+; W64-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
-; W64-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
-; W64-O0-NEXT: s_waitcnt vmcnt(4)
-; W64-O0-NEXT: v_readfirstlane_b32 s8, v0
; W64-O0-NEXT: s_waitcnt vmcnt(3)
+; W64-O0-NEXT: v_readfirstlane_b32 s8, v0
+; W64-O0-NEXT: s_waitcnt vmcnt(2)
; W64-O0-NEXT: v_readfirstlane_b32 s12, v1
; W64-O0-NEXT: s_mov_b32 s4, s8
; W64-O0-NEXT: s_mov_b32 s5, s12
; W64-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], s[4:5], v[0:1]
-; W64-O0-NEXT: s_waitcnt vmcnt(2)
-; W64-O0-NEXT: v_readfirstlane_b32 s7, v2
; W64-O0-NEXT: s_waitcnt vmcnt(1)
+; W64-O0-NEXT: v_readfirstlane_b32 s7, v2
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_readfirstlane_b32 s6, v3
; W64-O0-NEXT: s_mov_b32 s10, s7
; W64-O0-NEXT: s_mov_b32 s11, s6
@@ -589,7 +589,6 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; W64-O0-NEXT: s_mov_b32 s9, s12
; W64-O0-NEXT: s_mov_b32 s10, s7
; W64-O0-NEXT: s_mov_b32 s11, s6
-; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_writelane_b32 v17, s8, 3
; W64-O0-NEXT: v_writelane_b32 v17, s9, 4
; W64-O0-NEXT: v_writelane_b32 v17, s10, 5
@@ -601,7 +600,6 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; W64-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: ; %bb.2: ; in Loop: Header=BB1_1 Depth=1
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
; W64-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
@@ -613,7 +611,9 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; W64-O0-NEXT: v_readlane_b32 s10, v17, 5
; W64-O0-NEXT: v_readlane_b32 s11, v17, 6
; W64-O0-NEXT: v_readlane_b32 s6, v17, 0
-; W64-O0-NEXT: s_nop 4
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
+; W64-O0-NEXT: s_nop 2
; W64-O0-NEXT: buffer_load_format_x v0, v0, s[8:11], s6 idxen
; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
@@ -634,23 +634,23 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; W64-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: .LBB1_4: ; =>This Inner Loop Header: Depth=1
+; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
+; W64-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
-; W64-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
-; W64-O0-NEXT: s_waitcnt vmcnt(4)
-; W64-O0-NEXT: v_readfirstlane_b32 s8, v0
; W64-O0-NEXT: s_waitcnt vmcnt(3)
+; W64-O0-NEXT: v_readfirstlane_b32 s8, v0
+; W64-O0-NEXT: s_waitcnt vmcnt(2)
; W64-O0-NEXT: v_readfirstlane_b32 s12, v1
; W64-O0-NEXT: s_mov_b32 s4, s8
; W64-O0-NEXT: s_mov_b32 s5, s12
; W64-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], s[4:5], v[0:1]
-; W64-O0-NEXT: s_waitcnt vmcnt(2)
-; W64-O0-NEXT: v_readfirstlane_b32 s7, v2
; W64-O0-NEXT: s_waitcnt vmcnt(1)
+; W64-O0-NEXT: v_readfirstlane_b32 s7, v2
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_readfirstlane_b32 s6, v3
; W64-O0-NEXT: s_mov_b32 s10, s7
; W64-O0-NEXT: s_mov_b32 s11, s6
@@ -660,7 +660,6 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; W64-O0-NEXT: s_mov_b32 s9, s12
; W64-O0-NEXT: s_mov_b32 s10, s7
; W64-O0-NEXT: s_mov_b32 s11, s6
-; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_writelane_b32 v17, s8, 11
; W64-O0-NEXT: v_writelane_b32 v17, s9, 12
; W64-O0-NEXT: v_writelane_b32 v17, s10, 13
@@ -672,7 +671,6 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; W64-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: ; %bb.5: ; in Loop: Header=BB1_4 Depth=1
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
; W64-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
@@ -684,19 +682,15 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; W64-O0-NEXT: v_readlane_b32 s10, v17, 13
; W64-O0-NEXT: v_readlane_b32 s11, v17, 14
; W64-O0-NEXT: v_readlane_b32 s6, v17, 0
-; W64-O0-NEXT: s_nop 4
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
+; W64-O0-NEXT: s_nop 2
; W64-O0-NEXT: buffer_load_format_x v0, v0, s[8:11], s6 idxen
; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; W64-O0-NEXT: s_xor_b64 exec, exec, s[4:5]
; W64-O0-NEXT: s_cbranch_execnz .LBB1_4
; W64-O0-NEXT: ; %bb.6:
-; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
; W64-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
@@ -704,6 +698,13 @@ define void @mubuf_vgpr_adjacent_in_block(<4 x i32> %i, <4 x i32> %j, i32 %c, pt
; W64-O0-NEXT: v_readlane_b32 s4, v17, 9
; W64-O0-NEXT: v_readlane_b32 s5, v17, 10
; W64-O0-NEXT: s_mov_b64 exec, s[4:5]
+; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: global_store_dword v[3:4], v5, off
; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: global_store_dword v[0:1], v2, off
@@ -1084,23 +1085,23 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; W64-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
+; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
+; W64-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
-; W64-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
-; W64-O0-NEXT: s_waitcnt vmcnt(4)
-; W64-O0-NEXT: v_readfirstlane_b32 s8, v0
; W64-O0-NEXT: s_waitcnt vmcnt(3)
+; W64-O0-NEXT: v_readfirstlane_b32 s8, v0
+; W64-O0-NEXT: s_waitcnt vmcnt(2)
; W64-O0-NEXT: v_readfirstlane_b32 s12, v1
; W64-O0-NEXT: s_mov_b32 s4, s8
; W64-O0-NEXT: s_mov_b32 s5, s12
; W64-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], s[4:5], v[0:1]
-; W64-O0-NEXT: s_waitcnt vmcnt(2)
-; W64-O0-NEXT: v_readfirstlane_b32 s7, v2
; W64-O0-NEXT: s_waitcnt vmcnt(1)
+; W64-O0-NEXT: v_readfirstlane_b32 s7, v2
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_readfirstlane_b32 s6, v3
; W64-O0-NEXT: s_mov_b32 s10, s7
; W64-O0-NEXT: s_mov_b32 s11, s6
@@ -1110,7 +1111,6 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; W64-O0-NEXT: s_mov_b32 s9, s12
; W64-O0-NEXT: s_mov_b32 s10, s7
; W64-O0-NEXT: s_mov_b32 s11, s6
-; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_writelane_b32 v13, s8, 4
; W64-O0-NEXT: v_writelane_b32 v13, s9, 5
; W64-O0-NEXT: v_writelane_b32 v13, s10, 6
@@ -1122,7 +1122,6 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; W64-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: ; %bb.2: ; in Loop: Header=BB2_1 Depth=1
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
; W64-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
@@ -1134,15 +1133,15 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; W64-O0-NEXT: v_readlane_b32 s10, v13, 6
; W64-O0-NEXT: v_readlane_b32 s11, v13, 7
; W64-O0-NEXT: v_readlane_b32 s6, v13, 1
-; W64-O0-NEXT: s_nop 4
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
+; W64-O0-NEXT: s_nop 2
; W64-O0-NEXT: buffer_load_format_x v0, v0, s[8:11], s6 idxen
; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Spill
; W64-O0-NEXT: s_xor_b64 exec, exec, s[4:5]
; W64-O0-NEXT: s_cbranch_execnz .LBB2_1
; W64-O0-NEXT: ; %bb.3:
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
; W64-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
@@ -1151,7 +1150,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; W64-O0-NEXT: v_readlane_b32 s7, v13, 3
; W64-O0-NEXT: s_mov_b64 exec, s[6:7]
; W64-O0-NEXT: v_readlane_b32 s4, v13, 1
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b32 s5, 0x3ff
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_and_b32_e64 v1, v1, s5
; W64-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v1, s4
; W64-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
@@ -1182,23 +1184,23 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; W64-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: .LBB2_5: ; =>This Inner Loop Header: Depth=1
+; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
+; W64-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
-; W64-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
-; W64-O0-NEXT: s_waitcnt vmcnt(4)
-; W64-O0-NEXT: v_readfirstlane_b32 s8, v0
; W64-O0-NEXT: s_waitcnt vmcnt(3)
+; W64-O0-NEXT: v_readfirstlane_b32 s8, v0
+; W64-O0-NEXT: s_waitcnt vmcnt(2)
; W64-O0-NEXT: v_readfirstlane_b32 s12, v1
; W64-O0-NEXT: s_mov_b32 s4, s8
; W64-O0-NEXT: s_mov_b32 s5, s12
; W64-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], s[4:5], v[0:1]
-; W64-O0-NEXT: s_waitcnt vmcnt(2)
-; W64-O0-NEXT: v_readfirstlane_b32 s7, v2
; W64-O0-NEXT: s_waitcnt vmcnt(1)
+; W64-O0-NEXT: v_readfirstlane_b32 s7, v2
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_readfirstlane_b32 s6, v3
; W64-O0-NEXT: s_mov_b32 s10, s7
; W64-O0-NEXT: s_mov_b32 s11, s6
@@ -1208,7 +1210,6 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; W64-O0-NEXT: s_mov_b32 s9, s12
; W64-O0-NEXT: s_mov_b32 s10, s7
; W64-O0-NEXT: s_mov_b32 s11, s6
-; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_writelane_b32 v13, s8, 15
; W64-O0-NEXT: v_writelane_b32 v13, s9, 16
; W64-O0-NEXT: v_writelane_b32 v13, s10, 17
@@ -1220,7 +1221,6 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; W64-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: ; %bb.6: ; in Loop: Header=BB2_5 Depth=1
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
; W64-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
@@ -1232,14 +1232,15 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; W64-O0-NEXT: v_readlane_b32 s10, v13, 17
; W64-O0-NEXT: v_readlane_b32 s11, v13, 18
; W64-O0-NEXT: v_readlane_b32 s6, v13, 12
-; W64-O0-NEXT: s_nop 4
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
+; W64-O0-NEXT: s_nop 2
; W64-O0-NEXT: buffer_load_format_x v0, v0, s[8:11], s6 idxen
; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; W64-O0-NEXT: s_xor_b64 exec, exec, s[4:5]
; W64-O0-NEXT: s_cbranch_execnz .LBB2_5
; W64-O0-NEXT: ; %bb.7:
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
; W64-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
@@ -1247,12 +1248,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; W64-O0-NEXT: v_readlane_b32 s4, v13, 13
; W64-O0-NEXT: v_readlane_b32 s5, v13, 14
; W64-O0-NEXT: s_mov_b64 exec, s[4:5]
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Spill
; W64-O0-NEXT: .LBB2_8: ; %bb2
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_nop 0
-; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
; W64-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
@@ -1260,6 +1259,10 @@ define void @mubuf_vgpr_outside_entry(<4 x i32> %i, <4 x i32> %j, i32 %c, ptr ad
; W64-O0-NEXT: v_readlane_b32 s4, v13, 10
; W64-O0-NEXT: v_readlane_b32 s5, v13, 11
; W64-O0-NEXT: s_or_b64 exec, exec, s[4:5]
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: global_store_dword v[0:1], v2, off
; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
index 59ceecbf43b785..896cb6042e810b 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.ll
@@ -182,23 +182,23 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; W64-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: .LBB0_1: ; =>This Inner Loop Header: Depth=1
+; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
+; W64-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
-; W64-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
-; W64-O0-NEXT: s_waitcnt vmcnt(4)
-; W64-O0-NEXT: v_readfirstlane_b32 s8, v0
; W64-O0-NEXT: s_waitcnt vmcnt(3)
+; W64-O0-NEXT: v_readfirstlane_b32 s8, v0
+; W64-O0-NEXT: s_waitcnt vmcnt(2)
; W64-O0-NEXT: v_readfirstlane_b32 s12, v1
; W64-O0-NEXT: s_mov_b32 s4, s8
; W64-O0-NEXT: s_mov_b32 s5, s12
; W64-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], s[4:5], v[0:1]
-; W64-O0-NEXT: s_waitcnt vmcnt(2)
-; W64-O0-NEXT: v_readfirstlane_b32 s7, v2
; W64-O0-NEXT: s_waitcnt vmcnt(1)
+; W64-O0-NEXT: v_readfirstlane_b32 s7, v2
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_readfirstlane_b32 s6, v3
; W64-O0-NEXT: s_mov_b32 s10, s7
; W64-O0-NEXT: s_mov_b32 s11, s6
@@ -208,7 +208,6 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; W64-O0-NEXT: s_mov_b32 s9, s12
; W64-O0-NEXT: s_mov_b32 s10, s7
; W64-O0-NEXT: s_mov_b32 s11, s6
-; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_writelane_b32 v7, s8, 3
; W64-O0-NEXT: v_writelane_b32 v7, s9, 4
; W64-O0-NEXT: v_writelane_b32 v7, s10, 5
@@ -220,7 +219,6 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; W64-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: ; %bb.2: ; in Loop: Header=BB0_1 Depth=1
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
; W64-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
@@ -232,14 +230,15 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; W64-O0-NEXT: v_readlane_b32 s10, v7, 5
; W64-O0-NEXT: v_readlane_b32 s11, v7, 6
; W64-O0-NEXT: v_readlane_b32 s6, v7, 0
-; W64-O0-NEXT: s_nop 4
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
+; W64-O0-NEXT: s_nop 2
; W64-O0-NEXT: buffer_load_format_x v0, v0, s[8:11], s6 idxen
; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
; W64-O0-NEXT: s_xor_b64 exec, exec, s[4:5]
; W64-O0-NEXT: s_cbranch_execnz .LBB0_1
; W64-O0-NEXT: ; %bb.3:
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
; W64-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
@@ -247,6 +246,7 @@ define float @mubuf_vgpr(ptr addrspace(8) %i, i32 %c) #0 {
; W64-O0-NEXT: v_readlane_b32 s4, v7, 1
; W64-O0-NEXT: v_readlane_b32 s5, v7, 2
; W64-O0-NEXT: s_mov_b64 exec, s[4:5]
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; W64-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1
; W64-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[4:5]
@@ -598,23 +598,23 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; W64-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
+; W64-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
-; W64-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
-; W64-O0-NEXT: s_waitcnt vmcnt(4)
-; W64-O0-NEXT: v_readfirstlane_b32 s8, v0
; W64-O0-NEXT: s_waitcnt vmcnt(3)
+; W64-O0-NEXT: v_readfirstlane_b32 s8, v0
+; W64-O0-NEXT: s_waitcnt vmcnt(2)
; W64-O0-NEXT: v_readfirstlane_b32 s12, v1
; W64-O0-NEXT: s_mov_b32 s4, s8
; W64-O0-NEXT: s_mov_b32 s5, s12
; W64-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], s[4:5], v[0:1]
-; W64-O0-NEXT: s_waitcnt vmcnt(2)
-; W64-O0-NEXT: v_readfirstlane_b32 s7, v2
; W64-O0-NEXT: s_waitcnt vmcnt(1)
+; W64-O0-NEXT: v_readfirstlane_b32 s7, v2
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_readfirstlane_b32 s6, v3
; W64-O0-NEXT: s_mov_b32 s10, s7
; W64-O0-NEXT: s_mov_b32 s11, s6
@@ -624,7 +624,6 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; W64-O0-NEXT: s_mov_b32 s9, s12
; W64-O0-NEXT: s_mov_b32 s10, s7
; W64-O0-NEXT: s_mov_b32 s11, s6
-; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_writelane_b32 v17, s8, 3
; W64-O0-NEXT: v_writelane_b32 v17, s9, 4
; W64-O0-NEXT: v_writelane_b32 v17, s10, 5
@@ -636,7 +635,6 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; W64-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: ; %bb.2: ; in Loop: Header=BB1_1 Depth=1
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
; W64-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
@@ -648,7 +646,9 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; W64-O0-NEXT: v_readlane_b32 s10, v17, 5
; W64-O0-NEXT: v_readlane_b32 s11, v17, 6
; W64-O0-NEXT: v_readlane_b32 s6, v17, 0
-; W64-O0-NEXT: s_nop 4
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
+; W64-O0-NEXT: s_nop 2
; W64-O0-NEXT: buffer_load_format_x v0, v0, s[8:11], s6 idxen
; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
@@ -669,23 +669,23 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; W64-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: .LBB1_4: ; =>This Inner Loop Header: Depth=1
+; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
+; W64-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
-; W64-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
-; W64-O0-NEXT: s_waitcnt vmcnt(4)
-; W64-O0-NEXT: v_readfirstlane_b32 s8, v0
; W64-O0-NEXT: s_waitcnt vmcnt(3)
+; W64-O0-NEXT: v_readfirstlane_b32 s8, v0
+; W64-O0-NEXT: s_waitcnt vmcnt(2)
; W64-O0-NEXT: v_readfirstlane_b32 s12, v1
; W64-O0-NEXT: s_mov_b32 s4, s8
; W64-O0-NEXT: s_mov_b32 s5, s12
; W64-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], s[4:5], v[0:1]
-; W64-O0-NEXT: s_waitcnt vmcnt(2)
-; W64-O0-NEXT: v_readfirstlane_b32 s7, v2
; W64-O0-NEXT: s_waitcnt vmcnt(1)
+; W64-O0-NEXT: v_readfirstlane_b32 s7, v2
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_readfirstlane_b32 s6, v3
; W64-O0-NEXT: s_mov_b32 s10, s7
; W64-O0-NEXT: s_mov_b32 s11, s6
@@ -695,7 +695,6 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; W64-O0-NEXT: s_mov_b32 s9, s12
; W64-O0-NEXT: s_mov_b32 s10, s7
; W64-O0-NEXT: s_mov_b32 s11, s6
-; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_writelane_b32 v17, s8, 11
; W64-O0-NEXT: v_writelane_b32 v17, s9, 12
; W64-O0-NEXT: v_writelane_b32 v17, s10, 13
@@ -707,7 +706,6 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; W64-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: ; %bb.5: ; in Loop: Header=BB1_4 Depth=1
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
; W64-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
@@ -719,19 +717,15 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; W64-O0-NEXT: v_readlane_b32 s10, v17, 13
; W64-O0-NEXT: v_readlane_b32 s11, v17, 14
; W64-O0-NEXT: v_readlane_b32 s6, v17, 0
-; W64-O0-NEXT: s_nop 4
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
+; W64-O0-NEXT: s_nop 2
; W64-O0-NEXT: buffer_load_format_x v0, v0, s[8:11], s6 idxen
; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; W64-O0-NEXT: s_xor_b64 exec, exec, s[4:5]
; W64-O0-NEXT: s_cbranch_execnz .LBB1_4
; W64-O0-NEXT: ; %bb.6:
-; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
; W64-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
@@ -739,6 +733,13 @@ define void @mubuf_vgpr_adjacent_in_block(ptr addrspace(8) %i, ptr addrspace(8)
; W64-O0-NEXT: v_readlane_b32 s4, v17, 9
; W64-O0-NEXT: v_readlane_b32 s5, v17, 10
; W64-O0-NEXT: s_mov_b64 exec, s[4:5]
+; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: global_store_dword v[3:4], v5, off
; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: global_store_dword v[0:1], v2, off
@@ -1135,23 +1136,23 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; W64-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: .LBB2_1: ; =>This Inner Loop Header: Depth=1
+; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
+; W64-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
-; W64-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
-; W64-O0-NEXT: s_waitcnt vmcnt(4)
-; W64-O0-NEXT: v_readfirstlane_b32 s8, v0
; W64-O0-NEXT: s_waitcnt vmcnt(3)
+; W64-O0-NEXT: v_readfirstlane_b32 s8, v0
+; W64-O0-NEXT: s_waitcnt vmcnt(2)
; W64-O0-NEXT: v_readfirstlane_b32 s12, v1
; W64-O0-NEXT: s_mov_b32 s4, s8
; W64-O0-NEXT: s_mov_b32 s5, s12
; W64-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], s[4:5], v[0:1]
-; W64-O0-NEXT: s_waitcnt vmcnt(2)
-; W64-O0-NEXT: v_readfirstlane_b32 s7, v2
; W64-O0-NEXT: s_waitcnt vmcnt(1)
+; W64-O0-NEXT: v_readfirstlane_b32 s7, v2
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_readfirstlane_b32 s6, v3
; W64-O0-NEXT: s_mov_b32 s10, s7
; W64-O0-NEXT: s_mov_b32 s11, s6
@@ -1161,7 +1162,6 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; W64-O0-NEXT: s_mov_b32 s9, s12
; W64-O0-NEXT: s_mov_b32 s10, s7
; W64-O0-NEXT: s_mov_b32 s11, s6
-; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_writelane_b32 v13, s8, 4
; W64-O0-NEXT: v_writelane_b32 v13, s9, 5
; W64-O0-NEXT: v_writelane_b32 v13, s10, 6
@@ -1173,7 +1173,6 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; W64-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: ; %bb.2: ; in Loop: Header=BB2_1 Depth=1
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
; W64-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
@@ -1185,15 +1184,15 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; W64-O0-NEXT: v_readlane_b32 s10, v13, 6
; W64-O0-NEXT: v_readlane_b32 s11, v13, 7
; W64-O0-NEXT: v_readlane_b32 s6, v13, 1
-; W64-O0-NEXT: s_nop 4
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
+; W64-O0-NEXT: s_nop 2
; W64-O0-NEXT: buffer_load_format_x v0, v0, s[8:11], s6 idxen
; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Spill
; W64-O0-NEXT: s_xor_b64 exec, exec, s[4:5]
; W64-O0-NEXT: s_cbranch_execnz .LBB2_1
; W64-O0-NEXT: ; %bb.3:
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
; W64-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
@@ -1202,7 +1201,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; W64-O0-NEXT: v_readlane_b32 s7, v13, 3
; W64-O0-NEXT: s_mov_b64 exec, s[6:7]
; W64-O0-NEXT: v_readlane_b32 s4, v13, 1
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:68 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b32 s5, 0x3ff
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_and_b32_e64 v1, v1, s5
; W64-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v1, s4
; W64-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
@@ -1216,15 +1218,16 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; W64-O0-NEXT: s_mov_b64 exec, s[4:5]
; W64-O0-NEXT: s_cbranch_execz .LBB2_8
; W64-O0-NEXT: ; %bb.4: ; %bb1
-; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
; W64-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_readlane_b32 s4, v13, 0
+; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:20 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_mov_b32_e32 v6, v4
; W64-O0-NEXT: v_mov_b32_e32 v0, v3
; W64-O0-NEXT: v_mov_b32_e32 v4, v2
@@ -1253,23 +1256,23 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; W64-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: .LBB2_5: ; =>This Inner Loop Header: Depth=1
+; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
+; W64-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:80 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:84 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:88 ; 4-byte Folded Reload
; W64-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:92 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
-; W64-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
-; W64-O0-NEXT: s_waitcnt vmcnt(4)
-; W64-O0-NEXT: v_readfirstlane_b32 s8, v0
; W64-O0-NEXT: s_waitcnt vmcnt(3)
+; W64-O0-NEXT: v_readfirstlane_b32 s8, v0
+; W64-O0-NEXT: s_waitcnt vmcnt(2)
; W64-O0-NEXT: v_readfirstlane_b32 s12, v1
; W64-O0-NEXT: s_mov_b32 s4, s8
; W64-O0-NEXT: s_mov_b32 s5, s12
; W64-O0-NEXT: v_cmp_eq_u64_e64 s[4:5], s[4:5], v[0:1]
-; W64-O0-NEXT: s_waitcnt vmcnt(2)
-; W64-O0-NEXT: v_readfirstlane_b32 s7, v2
; W64-O0-NEXT: s_waitcnt vmcnt(1)
+; W64-O0-NEXT: v_readfirstlane_b32 s7, v2
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_readfirstlane_b32 s6, v3
; W64-O0-NEXT: s_mov_b32 s10, s7
; W64-O0-NEXT: s_mov_b32 s11, s6
@@ -1279,7 +1282,6 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; W64-O0-NEXT: s_mov_b32 s9, s12
; W64-O0-NEXT: s_mov_b32 s10, s7
; W64-O0-NEXT: s_mov_b32 s11, s6
-; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: v_writelane_b32 v13, s8, 15
; W64-O0-NEXT: v_writelane_b32 v13, s9, 16
; W64-O0-NEXT: v_writelane_b32 v13, s10, 17
@@ -1291,7 +1293,6 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; W64-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 ; 4-byte Folded Spill
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
; W64-O0-NEXT: ; %bb.6: ; in Loop: Header=BB2_5 Depth=1
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
; W64-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
@@ -1303,14 +1304,15 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; W64-O0-NEXT: v_readlane_b32 s10, v13, 17
; W64-O0-NEXT: v_readlane_b32 s11, v13, 18
; W64-O0-NEXT: v_readlane_b32 s6, v13, 12
-; W64-O0-NEXT: s_nop 4
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:76 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
+; W64-O0-NEXT: s_nop 2
; W64-O0-NEXT: buffer_load_format_x v0, v0, s[8:11], s6 idxen
; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
; W64-O0-NEXT: s_xor_b64 exec, exec, s[4:5]
; W64-O0-NEXT: s_cbranch_execnz .LBB2_5
; W64-O0-NEXT: ; %bb.7:
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
; W64-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
@@ -1318,12 +1320,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; W64-O0-NEXT: v_readlane_b32 s4, v13, 13
; W64-O0-NEXT: v_readlane_b32 s5, v13, 14
; W64-O0-NEXT: s_mov_b64 exec, s[4:5]
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:96 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; W64-O0-NEXT: .LBB2_8: ; %bb2
-; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; W64-O0-NEXT: s_nop 0
-; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; W64-O0-NEXT: s_or_saveexec_b64 s[16:17], -1
; W64-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 ; 4-byte Folded Reload
; W64-O0-NEXT: s_mov_b64 exec, s[16:17]
@@ -1331,6 +1331,10 @@ define void @mubuf_vgpr_outside_entry(ptr addrspace(8) %i, ptr addrspace(8) %j,
; W64-O0-NEXT: v_readlane_b32 s4, v13, 10
; W64-O0-NEXT: v_readlane_b32 s5, v13, 11
; W64-O0-NEXT: s_or_b64 exec, exec, s[4:5]
+; W64-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; W64-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: global_store_dword v[0:1], v2, off
; W64-O0-NEXT: s_waitcnt vmcnt(0)
; W64-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1
diff --git a/llvm/test/CodeGen/AMDGPU/rem_i128.ll b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
index f7f5bd56fa6f16..6583d5e8aa5a07 100644
--- a/llvm/test/CodeGen/AMDGPU/rem_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
@@ -622,10 +622,6 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB0_5
; GFX9-O0-NEXT: .LBB0_3: ; %Flow2
-; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23]
@@ -633,8 +629,13 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 4
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 5
; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
@@ -681,6 +682,13 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB0_3
; GFX9-O0-NEXT: .LBB0_5: ; %Flow1
+; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
+; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 8
+; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 9
+; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:136 ; 4-byte Folded Reload
@@ -689,15 +697,9 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:124 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
-; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23]
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 8
-; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 9
-; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:208 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:212 ; 4-byte Folded Spill
; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:200 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
@@ -711,6 +713,12 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_branch .LBB0_4
; GFX9-O0-NEXT: .LBB0_6: ; %udiv-do-while
; GFX9-O0-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
+; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: v_readlane_b32 s6, v30, 10
+; GFX9-O0-NEXT: v_readlane_b32 s7, v30, 11
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:240 ; 4-byte Folded Reload
@@ -735,13 +743,8 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
-; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23]
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT: v_readlane_b32 s6, v30, 10
-; GFX9-O0-NEXT: v_readlane_b32 s7, v30, 11
; GFX9-O0-NEXT: s_mov_b32 s4, 63
+; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: v_lshrrev_b64 v[28:29], s4, v[2:3]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v29
; GFX9-O0-NEXT: s_mov_b32 s5, 1
@@ -768,6 +771,7 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v1
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v29
+; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
; GFX9-O0-NEXT: v_mov_b32_e32 v10, v27
; GFX9-O0-NEXT: v_or3_b32 v6, v6, v7, v10
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0
@@ -777,6 +781,7 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v3
+; GFX9-O0-NEXT: s_waitcnt vmcnt(8)
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v25
; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v7
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2
@@ -788,10 +793,12 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v10, v5
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v22
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v23
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v13, v11
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v14
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v15
; GFX9-O0-NEXT: v_sub_co_u32_e32 v13, vcc, v13, v6
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_subb_co_u32_e32 v12, vcc, v12, v10, vcc
; GFX9-O0-NEXT: v_subb_co_u32_e32 v11, vcc, v11, v4, vcc
; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v7, v5, vcc
@@ -931,6 +938,9 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_cbranch_execnz .LBB0_6
; GFX9-O0-NEXT: s_branch .LBB0_1
; GFX9-O0-NEXT: .LBB0_7: ; %udiv-preheader
+; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
+; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23]
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
@@ -947,12 +957,9 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
-; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23]
-; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], v4, v[20:21]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-O0-NEXT: s_mov_b32 s6, 64
@@ -1033,7 +1040,6 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v14, s8
; GFX9-O0-NEXT: v_mov_b32_e32 v13, s7
; GFX9-O0-NEXT: v_mov_b32_e32 v12, s6
-; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-O0-NEXT: v_writelane_b32 v30, s4, 10
; GFX9-O0-NEXT: v_writelane_b32 v30, s5, 11
; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
@@ -1065,6 +1071,9 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:236 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB0_6
; GFX9-O0-NEXT: .LBB0_8: ; %udiv-bb1
+; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
+; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23]
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
@@ -1073,12 +1082,9 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
-; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_mov_b64 exec, s[22:23]
; GFX9-O0-NEXT: s_mov_b64 s[6:7], 1
; GFX9-O0-NEXT: s_mov_b32 s5, s6
-; GFX9-O0-NEXT: s_waitcnt vmcnt(2)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v0
; GFX9-O0-NEXT: s_mov_b32 s4, s7
; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0
@@ -1089,7 +1095,7 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s5
; GFX9-O0-NEXT: v_add_co_u32_e32 v8, vcc, v3, v4
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v4, vcc
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s8
; GFX9-O0-NEXT: v_addc_co_u32_e32 v0, vcc, v0, v4, vcc
@@ -1193,7 +1199,6 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_mov_b64 s[6:7], exec
; GFX9-O0-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5]
; GFX9-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7]
-; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 8
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 9
; GFX9-O0-NEXT: s_or_saveexec_b64 s[22:23], -1
@@ -1999,10 +2004,6 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB1_5
; GFX9-O0-NEXT: .LBB1_3: ; %Flow2
-; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
@@ -2010,8 +2011,13 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 2
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 3
; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:180 ; 4-byte Folded Spill
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:168 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
@@ -2058,6 +2064,13 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB1_3
; GFX9-O0-NEXT: .LBB1_5: ; %Flow1
+; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 6
+; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 7
+; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:128 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:132 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:120 ; 4-byte Folded Reload
@@ -2066,15 +2079,9 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:104 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
-; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 6
-; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 7
-; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:192 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: s_nop 0
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:196 ; 4-byte Folded Spill
; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:184 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
@@ -2088,6 +2095,12 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_branch .LBB1_4
; GFX9-O0-NEXT: .LBB1_6: ; %udiv-do-while
; GFX9-O0-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: v_readlane_b32 s6, v30, 8
+; GFX9-O0-NEXT: v_readlane_b32 s7, v30, 9
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:216 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:220 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:224 ; 4-byte Folded Reload
@@ -2112,13 +2125,8 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
-; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT: v_readlane_b32 s6, v30, 8
-; GFX9-O0-NEXT: v_readlane_b32 s7, v30, 9
; GFX9-O0-NEXT: s_mov_b32 s4, 63
+; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: v_lshrrev_b64 v[28:29], s4, v[2:3]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v29
; GFX9-O0-NEXT: s_mov_b32 s5, 1
@@ -2145,6 +2153,7 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v1
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v29
+; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
; GFX9-O0-NEXT: v_mov_b32_e32 v10, v27
; GFX9-O0-NEXT: v_or3_b32 v6, v6, v7, v10
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0
@@ -2154,6 +2163,7 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v3
+; GFX9-O0-NEXT: s_waitcnt vmcnt(8)
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v25
; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v7
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2
@@ -2165,10 +2175,12 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v10, v5
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v22
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v23
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v13, v11
; GFX9-O0-NEXT: v_mov_b32_e32 v11, v14
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v15
; GFX9-O0-NEXT: v_sub_co_u32_e32 v13, vcc, v13, v6
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_subb_co_u32_e32 v12, vcc, v12, v10, vcc
; GFX9-O0-NEXT: v_subb_co_u32_e32 v11, vcc, v11, v4, vcc
; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v7, v5, vcc
@@ -2308,6 +2320,9 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_cbranch_execnz .LBB1_6
; GFX9-O0-NEXT: s_branch .LBB1_1
; GFX9-O0-NEXT: .LBB1_7: ; %udiv-preheader
+; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
@@ -2324,12 +2339,9 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v20, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v21, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
-; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
-; GFX9-O0-NEXT: s_waitcnt vmcnt(10)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v10
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], v4, v[20:21]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-O0-NEXT: s_mov_b32 s6, 64
@@ -2410,7 +2422,6 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v14, s8
; GFX9-O0-NEXT: v_mov_b32_e32 v13, s7
; GFX9-O0-NEXT: v_mov_b32_e32 v12, s6
-; GFX9-O0-NEXT: s_waitcnt vmcnt(4)
; GFX9-O0-NEXT: v_writelane_b32 v30, s4, 8
; GFX9-O0-NEXT: v_writelane_b32 v30, s5, 9
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
@@ -2442,6 +2453,9 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB1_6
; GFX9-O0-NEXT: .LBB1_8: ; %udiv-bb1
+; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
+; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
@@ -2450,12 +2464,9 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
-; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 s[6:7], 1
; GFX9-O0-NEXT: s_mov_b32 s5, s6
-; GFX9-O0-NEXT: s_waitcnt vmcnt(2)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v0
; GFX9-O0-NEXT: s_mov_b32 s4, s7
; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0
@@ -2466,7 +2477,7 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s5
; GFX9-O0-NEXT: v_add_co_u32_e32 v8, vcc, v3, v4
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-O0-NEXT: s_waitcnt vmcnt(1)
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v4, vcc
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s8
; GFX9-O0-NEXT: v_addc_co_u32_e32 v0, vcc, v0, v4, vcc
@@ -2570,7 +2581,6 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_mov_b64 s[6:7], exec
; GFX9-O0-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5]
; GFX9-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7]
-; GFX9-O0-NEXT: s_waitcnt vmcnt(16)
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 6
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 7
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
diff --git a/llvm/test/CodeGen/AMDGPU/trap-abis.ll b/llvm/test/CodeGen/AMDGPU/trap-abis.ll
index 4dfd4c095c87a0..0daa6860072616 100644
--- a/llvm/test/CodeGen/AMDGPU/trap-abis.ll
+++ b/llvm/test/CodeGen/AMDGPU/trap-abis.ll
@@ -367,14 +367,15 @@ define amdgpu_kernel void @trap_with_use_after(ptr addrspace(1) %arg0, ptr addrs
; HSA-TRAP-GFX1100-O0-NEXT: scratch_store_b32 off, v0, off offset:4 ; 4-byte Folded Spill
; HSA-TRAP-GFX1100-O0-NEXT: s_cbranch_execnz .LBB2_2
; HSA-TRAP-GFX1100-O0-NEXT: ; %bb.1:
-; HSA-TRAP-GFX1100-O0-NEXT: scratch_load_b32 v0, off, off offset:8 ; 4-byte Folded Reload
-; HSA-TRAP-GFX1100-O0-NEXT: scratch_load_b32 v1, off, off offset:4 ; 4-byte Folded Reload
; HSA-TRAP-GFX1100-O0-NEXT: s_or_saveexec_b32 s6, -1
; HSA-TRAP-GFX1100-O0-NEXT: scratch_load_b32 v2, off, off ; 4-byte Folded Reload
; HSA-TRAP-GFX1100-O0-NEXT: s_mov_b32 exec_lo, s6
; HSA-TRAP-GFX1100-O0-NEXT: s_waitcnt vmcnt(0)
; HSA-TRAP-GFX1100-O0-NEXT: v_readlane_b32 s0, v2, 0
; HSA-TRAP-GFX1100-O0-NEXT: v_readlane_b32 s1, v2, 1
+; HSA-TRAP-GFX1100-O0-NEXT: scratch_load_b32 v0, off, off offset:8 ; 4-byte Folded Reload
+; HSA-TRAP-GFX1100-O0-NEXT: scratch_load_b32 v1, off, off offset:4 ; 4-byte Folded Reload
+; HSA-TRAP-GFX1100-O0-NEXT: s_waitcnt vmcnt(0)
; HSA-TRAP-GFX1100-O0-NEXT: global_store_b32 v0, v1, s[0:1] dlc
; HSA-TRAP-GFX1100-O0-NEXT: s_waitcnt_vscnt null, 0x0
; HSA-TRAP-GFX1100-O0-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-spill-placement-issue61083.ll b/llvm/test/CodeGen/AMDGPU/vgpr-spill-placement-issue61083.ll
index a827ebe96cfcf4..2c9b53b46c098e 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-spill-placement-issue61083.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-spill-placement-issue61083.ll
@@ -35,7 +35,6 @@ define amdgpu_kernel void @__omp_offloading_16_dd2df_main_l9() {
; CHECK-NEXT: s_cbranch_execz .LBB0_2
; CHECK-NEXT: ; %bb.1: ; %bb193
; CHECK-NEXT: .LBB0_2: ; %bb194
-; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], 0 offset:4 ; 4-byte Folded Reload
; CHECK-NEXT: s_or_saveexec_b64 s[8:9], -1
; CHECK-NEXT: buffer_load_dword v3, off, s[0:3], 0 ; 4-byte Folded Reload
; CHECK-NEXT: s_mov_b64 exec, s[8:9]
@@ -43,7 +42,9 @@ define amdgpu_kernel void @__omp_offloading_16_dd2df_main_l9() {
; CHECK-NEXT: v_readlane_b32 s4, v3, 0
; CHECK-NEXT: v_readlane_b32 s5, v3, 1
; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
+; CHECK-NEXT: buffer_load_dword v0, off, s[0:3], 0 offset:4 ; 4-byte Folded Reload
; CHECK-NEXT: s_mov_b32 s4, 0xffff
+; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: v_and_b32_e64 v0, s4, v0
; CHECK-NEXT: s_mov_b32 s4, 0
; CHECK-NEXT: v_cmp_ne_u32_e64 s[4:5], v0, s4
diff --git a/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll b/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll
index 025381d5c16df8..1089093ea691c3 100644
--- a/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll
+++ b/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll
@@ -206,9 +206,6 @@ define amdgpu_gfx void @strict_wwm_cfg(ptr addrspace(8) inreg %tmp14, i32 %arg)
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v1
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX9-O0-NEXT: .LBB1_2: ; %merge
-; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_nop 0
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_or_saveexec_b64 s[46:47], -1
; GFX9-O0-NEXT: buffer_load_dword v5, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[46:47]
@@ -220,6 +217,9 @@ define amdgpu_gfx void @strict_wwm_cfg(ptr addrspace(8) inreg %tmp14, i32 %arg)
; GFX9-O0-NEXT: v_readlane_b32 s39, v5, 1
; GFX9-O0-NEXT: v_readlane_b32 s34, v5, 2
; GFX9-O0-NEXT: v_readlane_b32 s35, v5, 3
+; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[36:37], v0, v3
; GFX9-O0-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[36:37]
; GFX9-O0-NEXT: s_mov_b32 s36, 1
diff --git a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
index 312628c7b5451e..027081752a11bb 100644
--- a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
+++ b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
@@ -183,9 +183,6 @@ define amdgpu_cs void @cfg(ptr addrspace(8) inreg %tmp14, i32 %arg) {
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v1
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[16:19], 0 offset:4 ; 4-byte Folded Spill
; GFX9-O0-NEXT: .LBB1_2: ; %merge
-; GFX9-O0-NEXT: buffer_load_dword v0, off, s[16:19], 0 offset:8 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_nop 0
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[16:19], 0 offset:4 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_or_saveexec_b64 s[12:13], -1
; GFX9-O0-NEXT: buffer_load_dword v5, off, s[16:19], 0 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[12:13]
@@ -197,6 +194,9 @@ define amdgpu_cs void @cfg(ptr addrspace(8) inreg %tmp14, i32 %arg) {
; GFX9-O0-NEXT: v_readlane_b32 s3, v5, 2
; GFX9-O0-NEXT: v_readlane_b32 s0, v5, 3
; GFX9-O0-NEXT: v_readlane_b32 s1, v5, 4
+; GFX9-O0-NEXT: buffer_load_dword v0, off, s[16:19], 0 offset:8 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[16:19], 0 offset:4 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], v0, v3
; GFX9-O0-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
; GFX9-O0-NEXT: s_mov_b32 s4, 1
@@ -1035,9 +1035,6 @@ define amdgpu_cs void @strict_wwm_cfg(ptr addrspace(8) inreg %tmp14, i32 %arg) {
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v1
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[16:19], 0 offset:4 ; 4-byte Folded Spill
; GFX9-O0-NEXT: .LBB8_2: ; %merge
-; GFX9-O0-NEXT: buffer_load_dword v0, off, s[16:19], 0 offset:8 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: s_nop 0
-; GFX9-O0-NEXT: buffer_load_dword v3, off, s[16:19], 0 offset:4 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_or_saveexec_b64 s[12:13], -1
; GFX9-O0-NEXT: buffer_load_dword v5, off, s[16:19], 0 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[12:13]
@@ -1049,6 +1046,9 @@ define amdgpu_cs void @strict_wwm_cfg(ptr addrspace(8) inreg %tmp14, i32 %arg) {
; GFX9-O0-NEXT: v_readlane_b32 s3, v5, 2
; GFX9-O0-NEXT: v_readlane_b32 s0, v5, 3
; GFX9-O0-NEXT: v_readlane_b32 s1, v5, 4
+; GFX9-O0-NEXT: buffer_load_dword v0, off, s[16:19], 0 offset:8 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v3, off, s[16:19], 0 offset:4 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[4:5], v0, v3
; GFX9-O0-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
; GFX9-O0-NEXT: s_mov_b32 s4, 1
More information about the llvm-commits
mailing list