[llvm] 7ae4aab - [SelectionDAG] Take phi op divergence from the op not the phi (#220853)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 30 10:50:17 PDT 2026
Author: Jay Foad
Date: 2026-09-30T17:44:45Z
New Revision: 7ae4aabf6d49ac1a3a5a9a6d7bc0e8778d66ee43
URL: https://github.com/llvm/llvm-project/commit/7ae4aabf6d49ac1a3a5a9a6d7bc0e8778d66ee43
DIFF: https://github.com/llvm/llvm-project/commit/7ae4aabf6d49ac1a3a5a9a6d7bc0e8778d66ee43.diff
LOG: [SelectionDAG] Take phi op divergence from the op not the phi (#220853)
When creating registers for phi operands, take the type and the
IsDivergent bit from the operand not from the result of the phi node.
In practice this only affects AMDGPU and only affects constants used as
operands to a divergent phi. The effect is to put the constant into an
SGPR even if the result of the phi is a VGPR.
As well as fixing a correctness problem that arose when enabling
-structurizecfg-skip-uniform-regions, this has a small overall effect in
increasing SALU usage but reducing VALU usage. Overall register usage is
generally unaffected, except in some regalloc stress tests.
Co-authored-by: Claude Opus 5 (1M context) <noreply at anthropic.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply at anthropic.com>
Added:
llvm/test/CodeGen/AMDGPU/phi-constant-operand-regclass.ll
Modified:
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll
llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-flat.ll
llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll
llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
llvm/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll
llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll
llvm/test/CodeGen/AMDGPU/dagcombine-fma-crash.ll
llvm/test/CodeGen/AMDGPU/div_i128.ll
llvm/test/CodeGen/AMDGPU/div_v2i128.ll
llvm/test/CodeGen/AMDGPU/divergent-branch-uniform-condition.ll
llvm/test/CodeGen/AMDGPU/expand-mov-b64-globaladdr.ll
llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-nondeterminism.ll
llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-phi-regression-issue130646-issue130119.ll
llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-sgpr32-to-vgpr16.ll
llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-rtn.ll
llvm/test/CodeGen/AMDGPU/implicit-def-muse.ll
llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.ll
llvm/test/CodeGen/AMDGPU/loop_break.ll
llvm/test/CodeGen/AMDGPU/machine-sink-loop-var-out-of-divergent-loop-swdev407790.ll
llvm/test/CodeGen/AMDGPU/mad_int24.ll
llvm/test/CodeGen/AMDGPU/masked-load-vectortypes.ll
llvm/test/CodeGen/AMDGPU/mfma-loop.ll
llvm/test/CodeGen/AMDGPU/mmra.ll
llvm/test/CodeGen/AMDGPU/multilevel-break.ll
llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
llvm/test/CodeGen/AMDGPU/permute.ll
llvm/test/CodeGen/AMDGPU/phi-av-pressure.ll
llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
llvm/test/CodeGen/AMDGPU/reg-coalescer-sched-crash.ll
llvm/test/CodeGen/AMDGPU/rem_i128.ll
llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-spill-cost-reset.ll
llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-reserved-wwm-vgpr.ll
llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-multi-store-codegen.ll
llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-multi-store.ll
llvm/test/CodeGen/AMDGPU/scheduler-rp-calc-one-successor-two-predecessors-bug.ll
llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
llvm/test/CodeGen/AMDGPU/set-inactive-wwm-overwrite.ll
llvm/test/CodeGen/AMDGPU/sgpr-control-flow.ll
llvm/test/CodeGen/AMDGPU/sgpr-scavenge-fi-stack-id.ll
llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll
llvm/test/CodeGen/AMDGPU/structurize-hoist.ll
llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll
llvm/test/CodeGen/AMDGPU/undef-handling-crash-in-ra.ll
llvm/test/CodeGen/AMDGPU/uniform_branch_with_floating_point_cond.ll
llvm/test/CodeGen/AMDGPU/use-after-free-after-cleanup-failed-vreg.ll
llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll
llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.ll
llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll
llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll
llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
Removed:
################################################################################
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index f7eec8c5088d4..cf001a05fb726 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -12507,7 +12507,7 @@ SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
if (const auto *C = dyn_cast<Constant>(PHIOp)) {
Register &RegOut = ConstantsOut[C];
if (!RegOut) {
- RegOut = FuncInfo.CreateRegs(&PN);
+ RegOut = FuncInfo.CreateRegs(PHIOp);
// We need to zero/sign extend ConstantInt phi operands to match
// assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo.
ISD::NodeType ExtendType = ISD::ANY_EXTEND;
@@ -12525,7 +12525,7 @@ SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
assert(isa<AllocaInst>(PHIOp) &&
FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&
"Didn't codegen value into a register!??");
- Reg = FuncInfo.CreateRegs(&PN);
+ Reg = FuncInfo.CreateRegs(PHIOp);
CopyValueToVirtualRegister(PHIOp, Reg);
}
}
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
index cfc9faa7c5d18..ebf17d2d6c414 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-pregfx12.ll
@@ -370,9 +370,9 @@ define void @test_pipelined_loop(ptr addrspace(1) %foo, ptr addrspace(3) %lds, p
; SDAG-NEXT: global_load_dword v[0:1], off lds
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: global_load_dword v[0:1], off lds
-; SDAG-NEXT: v_mov_b32_e32 v5, 0
; SDAG-NEXT: s_mov_b32 s6, 2
; SDAG-NEXT: s_mov_b64 s[4:5], 0
+; SDAG-NEXT: v_mov_b32_e32 v5, 0
; SDAG-NEXT: ; asyncmark
; SDAG-NEXT: .LBB4_1: ; %loop_body
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
diff --git a/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
index 1d4ada34650c3..3a17596b5136a 100644
--- a/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
@@ -14,9 +14,9 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: s_cselect_b64 s[14:15], -1, 0
; CHECK-NEXT: s_xor_b64 s[16:17], s[14:15], -1
; CHECK-NEXT: s_bitcmp1_b32 s0, 8
-; CHECK-NEXT: v_accvgpr_write_b32 a4, v1 ; Reload Reuse
; CHECK-NEXT: s_cselect_b64 s[18:19], -1, 0
-; CHECK-NEXT: v_accvgpr_write_b32 a5, v0 ; Reload Reuse
+; CHECK-NEXT: v_accvgpr_write_b32 a3, v1
+; CHECK-NEXT: v_accvgpr_write_b32 a2, v0
; CHECK-NEXT: s_xor_b64 s[20:21], s[18:19], -1
; CHECK-NEXT: v_mov_b32_e32 v2, 0xa17f65f6
; CHECK-NEXT: v_mov_b32_e32 v3, 0xbe927e4f
@@ -41,130 +41,127 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: v_mov_b32_e32 v19, 0xbfd55555
; CHECK-NEXT: s_and_b64 s[4:5], exec, s[16:17]
; CHECK-NEXT: v_mov_b32_e32 v0, 0
-; CHECK-NEXT: v_mov_b64_e32 v[30:31], 0
-; CHECK-NEXT: ; implicit-def: $agpr0_agpr1
-; CHECK-NEXT: ; implicit-def: $vgpr22_vgpr23
+; CHECK-NEXT: v_mov_b64_e32 v[28:29], 0
+; CHECK-NEXT: ; implicit-def: $vgpr30_vgpr31
+; CHECK-NEXT: ; implicit-def: $sgpr24_sgpr25
; CHECK-NEXT: s_branch .LBB0_2
; CHECK-NEXT: .LBB0_1: ; %Flow9
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: s_and_b64 s[6:7], s[24:25], exec
+; CHECK-NEXT: s_and_b64 s[6:7], s[26:27], exec
; CHECK-NEXT: s_cselect_b32 s6, 1, 0
; CHECK-NEXT: s_cmp_lg_u32 s6, 1
; CHECK-NEXT: s_cbranch_scc0 .LBB0_17
; CHECK-NEXT: .LBB0_2: ; %._crit_edge1942.i.i.i3548
; CHECK-NEXT: ; =>This Loop Header: Depth=1
-; CHECK-NEXT: ; Child Loop BB0_6 Depth 2
+; CHECK-NEXT: ; Child Loop BB0_7 Depth 2
; CHECK-NEXT: s_and_b64 s[6:7], exec, s[18:19]
; CHECK-NEXT: s_cselect_b32 s6, 1, 0
; CHECK-NEXT: s_cmp_lg_u32 s6, 1
-; CHECK-NEXT: s_cbranch_scc1 .LBB0_9
+; CHECK-NEXT: s_cbranch_scc1 .LBB0_10
; CHECK-NEXT: ; %bb.3: ; %.preheader1868.i.i.i3244
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: s_mov_b64 vcc, s[0:1]
-; CHECK-NEXT: s_cbranch_vccz .LBB0_10
+; CHECK-NEXT: s_cbranch_vccz .LBB0_11
; CHECK-NEXT: ; %bb.4: ; %.preheader1855.i.i.i3329.preheader
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[10:11]
-; CHECK-NEXT: flat_load_dwordx2 v[24:25], v[24:25]
-; CHECK-NEXT: v_accvgpr_read_b32 v27, a4 ; Reload Reuse
-; CHECK-NEXT: v_accvgpr_read_b32 v26, a5 ; Reload Reuse
-; CHECK-NEXT: v_mov_b64_e32 v[28:29], v[2:3]
+; CHECK-NEXT: v_mov_b64_e32 v[22:23], s[10:11]
+; CHECK-NEXT: flat_load_dwordx2 v[22:23], v[22:23]
+; CHECK-NEXT: v_accvgpr_read_b32 v25, a3
+; CHECK-NEXT: v_accvgpr_read_b32 v24, a2
+; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[2:3]
; CHECK-NEXT: v_mov_b64_e32 v[16:17], v[20:21]
-; CHECK-NEXT: v_mov_b64_e32 v[30:31], 0
-; CHECK-NEXT: v_accvgpr_write_b32 a2, 0
-; CHECK-NEXT: v_accvgpr_write_b32 a3, 0
+; CHECK-NEXT: v_mov_b64_e32 v[28:29], 0
+; CHECK-NEXT: v_accvgpr_write_b32 a0, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a1, 0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT: v_fmac_f64_e32 v[24:25], 0, v[22:23]
; CHECK-NEXT: v_fmac_f64_e32 v[26:27], 0, v[24:25]
-; CHECK-NEXT: v_fmac_f64_e32 v[28:29], 0, v[26:27]
-; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[4:5]
-; CHECK-NEXT: v_fmac_f64_e32 v[26:27], 0, v[28:29]
-; CHECK-NEXT: v_mov_b64_e32 v[28:29], v[6:7]
-; CHECK-NEXT: v_fmac_f64_e32 v[28:29], 0, v[26:27]
-; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[8:9]
-; CHECK-NEXT: v_fmac_f64_e32 v[26:27], 0, v[28:29]
-; CHECK-NEXT: v_mov_b64_e32 v[28:29], v[10:11]
-; CHECK-NEXT: v_fmac_f64_e32 v[28:29], 0, v[26:27]
-; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[12:13]
-; CHECK-NEXT: v_fmac_f64_e32 v[26:27], 0, v[28:29]
-; CHECK-NEXT: v_mov_b64_e32 v[28:29], v[14:15]
-; CHECK-NEXT: v_fmac_f64_e32 v[28:29], 0, v[26:27]
-; CHECK-NEXT: v_fmac_f64_e32 v[16:17], 0, v[28:29]
-; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[18:19]
-; CHECK-NEXT: v_fmac_f64_e32 v[26:27], 0, v[16:17]
-; CHECK-NEXT: s_branch .LBB0_6
-; CHECK-NEXT: .LBB0_5: ; %Flow
-; CHECK-NEXT: ; in Loop: Header=BB0_6 Depth=2
+; CHECK-NEXT: v_mov_b64_e32 v[24:25], v[4:5]
+; CHECK-NEXT: v_fmac_f64_e32 v[24:25], 0, v[26:27]
+; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[6:7]
+; CHECK-NEXT: v_fmac_f64_e32 v[26:27], 0, v[24:25]
+; CHECK-NEXT: v_mov_b64_e32 v[24:25], v[8:9]
+; CHECK-NEXT: v_fmac_f64_e32 v[24:25], 0, v[26:27]
+; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[10:11]
+; CHECK-NEXT: v_fmac_f64_e32 v[26:27], 0, v[24:25]
+; CHECK-NEXT: v_mov_b64_e32 v[24:25], v[12:13]
+; CHECK-NEXT: v_fmac_f64_e32 v[24:25], 0, v[26:27]
+; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[14:15]
+; CHECK-NEXT: v_fmac_f64_e32 v[26:27], 0, v[24:25]
+; CHECK-NEXT: v_fmac_f64_e32 v[16:17], 0, v[26:27]
+; CHECK-NEXT: v_mov_b64_e32 v[24:25], v[18:19]
+; CHECK-NEXT: v_fmac_f64_e32 v[24:25], 0, v[16:17]
+; CHECK-NEXT: s_branch .LBB0_7
+; CHECK-NEXT: .LBB0_5: ; in Loop: Header=BB0_7 Depth=2
+; CHECK-NEXT: ; implicit-def: $agpr0_agpr1
+; CHECK-NEXT: s_mov_b64 s[6:7], -1
+; CHECK-NEXT: .LBB0_6: ; %Flow
+; CHECK-NEXT: ; in Loop: Header=BB0_7 Depth=2
; CHECK-NEXT: s_and_b64 vcc, exec, s[6:7]
-; CHECK-NEXT: s_cbranch_vccnz .LBB0_11
-; CHECK-NEXT: .LBB0_6: ; %.preheader1855.i.i.i3329
+; CHECK-NEXT: s_cbranch_vccnz .LBB0_12
+; CHECK-NEXT: .LBB0_7: ; %.preheader1855.i.i.i3329
; CHECK-NEXT: ; Parent Loop BB0_2 Depth=1
; CHECK-NEXT: ; => This Inner Loop Header: Depth=2
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a3
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a2
-; CHECK-NEXT: s_mov_b64 s[24:25], -1
-; CHECK-NEXT: s_mov_b64 s[6:7], -1
+; CHECK-NEXT: v_accvgpr_read_b32 v27, a1
+; CHECK-NEXT: v_accvgpr_read_b32 v26, a0
+; CHECK-NEXT: s_mov_b64 s[26:27], -1
; CHECK-NEXT: s_mov_b64 vcc, s[2:3]
-; CHECK-NEXT: ; implicit-def: $agpr2_agpr3
; CHECK-NEXT: s_cbranch_vccz .LBB0_5
-; CHECK-NEXT: ; %bb.7: ; %.lr.ph2070.i.i.i3291
-; CHECK-NEXT: ; in Loop: Header=BB0_6 Depth=2
-; CHECK-NEXT: v_accvgpr_mov_b32 a3, a1
-; CHECK-NEXT: v_accvgpr_mov_b32 a2, a0
+; CHECK-NEXT: ; %bb.8: ; %.lr.ph2070.i.i.i3291
+; CHECK-NEXT: ; in Loop: Header=BB0_7 Depth=2
+; CHECK-NEXT: v_accvgpr_write_b32 a0, v30
+; CHECK-NEXT: v_accvgpr_write_b32 a1, v31
; CHECK-NEXT: s_mov_b64 s[6:7], s[16:17]
; CHECK-NEXT: s_mov_b64 vcc, s[4:5]
-; CHECK-NEXT: s_cbranch_vccz .LBB0_5
-; CHECK-NEXT: ; %bb.8: ; %.preheader1856.preheader.i.i.i3325
-; CHECK-NEXT: ; in Loop: Header=BB0_6 Depth=2
-; CHECK-NEXT: s_mov_b64 s[24:25], 0
-; CHECK-NEXT: v_accvgpr_write_b32 a2, v26
-; CHECK-NEXT: v_accvgpr_write_b32 a3, v27
-; CHECK-NEXT: s_mov_b64 s[6:7], 0
-; CHECK-NEXT: s_branch .LBB0_5
-; CHECK-NEXT: .LBB0_9: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[12:13]
-; CHECK-NEXT: s_mov_b64 s[22:23], 0
+; CHECK-NEXT: s_cbranch_vccz .LBB0_6
+; CHECK-NEXT: ; %bb.9: ; %.preheader1856.preheader.i.i.i3325
+; CHECK-NEXT: ; in Loop: Header=BB0_7 Depth=2
+; CHECK-NEXT: s_mov_b64 s[26:27], 0
; CHECK-NEXT: v_accvgpr_write_b32 a0, v24
; CHECK-NEXT: v_accvgpr_write_b32 a1, v25
+; CHECK-NEXT: s_mov_b64 s[6:7], 0
+; CHECK-NEXT: s_branch .LBB0_6
+; CHECK-NEXT: .LBB0_10: ; in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT: s_mov_b64 s[22:23], 0
+; CHECK-NEXT: v_mov_b64_e32 v[30:31], s[12:13]
; CHECK-NEXT: s_mov_b64 s[6:7], s[20:21]
; CHECK-NEXT: s_branch .LBB0_15
-; CHECK-NEXT: .LBB0_10: ; in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT: .LBB0_11: ; in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: s_mov_b64 s[6:7], -1
-; CHECK-NEXT: v_mov_b64_e32 v[22:23], 0
+; CHECK-NEXT: s_mov_b64 s[24:25], 0
; CHECK-NEXT: s_branch .LBB0_15
-; CHECK-NEXT: .LBB0_11: ; %loop.exit.guard
+; CHECK-NEXT: .LBB0_12: ; %loop.exit.guard
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: s_and_b64 vcc, exec, s[24:25]
-; CHECK-NEXT: s_cbranch_vccz .LBB0_13
-; CHECK-NEXT: ; %bb.12: ; %._crit_edge2105.i.i.i2330.loopexit
+; CHECK-NEXT: s_and_b64 vcc, exec, s[26:27]
+; CHECK-NEXT: s_cbranch_vccz .LBB0_14
+; CHECK-NEXT: ; %bb.13: ; %._crit_edge2105.i.i.i2330.loopexit
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: v_cmp_nlg_f64_e64 s[6:7], 0, v[28:29]
-; CHECK-NEXT: v_cndmask_b32_e64 v23, v23, 0, s[14:15]
-; CHECK-NEXT: v_cndmask_b32_e64 v22, v22, 0, s[14:15]
+; CHECK-NEXT: v_cmp_nlg_f64_e64 s[6:7], 0, v[26:27]
+; CHECK-NEXT: v_mov_b64_e32 v[28:29], 0
+; CHECK-NEXT: v_mov_b64_e32 v[30:31], v[22:23]
; CHECK-NEXT: v_cndmask_b32_e64 v16, 0, 1, s[6:7]
; CHECK-NEXT: v_mov_b32_e32 v17, v16
; CHECK-NEXT: s_and_b64 s[6:7], exec, s[14:15]
+; CHECK-NEXT: s_cselect_b32 s25, 0, s25
+; CHECK-NEXT: s_cselect_b32 s24, 0, s24
; CHECK-NEXT: global_store_dwordx2 v0, v[16:17], s[8:9]
; CHECK-NEXT: s_cselect_b32 s23, s23, 0
; CHECK-NEXT: s_cselect_b32 s22, s22, 0
; CHECK-NEXT: s_mov_b64 s[6:7], -1
-; CHECK-NEXT: v_mov_b64_e32 v[30:31], 0
-; CHECK-NEXT: s_branch .LBB0_14
-; CHECK-NEXT: .LBB0_13: ; in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT: s_branch .LBB0_15
+; CHECK-NEXT: .LBB0_14: ; in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: s_mov_b64 s[6:7], 0
-; CHECK-NEXT: v_mov_b64_e32 v[22:23], 0
-; CHECK-NEXT: .LBB0_14: ; %Flow6
-; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: v_accvgpr_write_b32 a0, v24
-; CHECK-NEXT: v_accvgpr_write_b32 a1, v25
+; CHECK-NEXT: s_mov_b64 s[24:25], 0
+; CHECK-NEXT: v_mov_b64_e32 v[30:31], v[22:23]
; CHECK-NEXT: .LBB0_15: ; %Flow6
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: s_mov_b64 s[24:25], -1
+; CHECK-NEXT: s_mov_b64 s[26:27], -1
; CHECK-NEXT: s_and_b64 vcc, exec, s[6:7]
; CHECK-NEXT: s_cbranch_vccz .LBB0_1
; CHECK-NEXT: ; %bb.16: ; %._crit_edge2105.i.i.i2330
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: s_mov_b64 s[24:25], 0
-; CHECK-NEXT: global_store_dwordx2 v0, v[30:31], s[8:9]
+; CHECK-NEXT: s_mov_b64 s[26:27], 0
+; CHECK-NEXT: global_store_dwordx2 v0, v[28:29], s[8:9]
; CHECK-NEXT: s_branch .LBB0_1
; CHECK-NEXT: .LBB0_17: ; %DummyReturnBlock
; CHECK-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll b/llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll
index 7694e30a612af..1cb404b0ff56e 100644
--- a/llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/bb-prolog-spill-during-regalloc.ll
@@ -8,11 +8,13 @@ define i32 @prolog_spill(i32 %arg0, i32 %arg1, i32 %arg2) {
; REGALLOC-NEXT: successors: %bb.3(0x40000000), %bb.1(0x40000000)
; REGALLOC-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2
; REGALLOC-NEXT: {{ $}}
- ; REGALLOC-NEXT: SI_SPILL_V32_SAVE killed $vgpr2, %stack.4, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.4, addrspace 5)
- ; REGALLOC-NEXT: SI_SPILL_V32_SAVE killed $vgpr1, %stack.3, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.3, addrspace 5)
+ ; REGALLOC-NEXT: SI_SPILL_V32_SAVE killed $vgpr2, %stack.5, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.5, addrspace 5)
+ ; REGALLOC-NEXT: SI_SPILL_V32_SAVE killed $vgpr1, %stack.4, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.4, addrspace 5)
; REGALLOC-NEXT: renamable $sgpr4 = S_MOV_B32 50
; REGALLOC-NEXT: renamable $sgpr4_sgpr5 = V_CMP_GE_I32_e64 killed $vgpr0, killed $sgpr4, implicit $exec
- ; REGALLOC-NEXT: renamable $vgpr0 = IMPLICIT_DEF
+ ; REGALLOC-NEXT: renamable $sgpr6 = IMPLICIT_DEF
+ ; REGALLOC-NEXT: renamable $vgpr0 = COPY killed renamable $sgpr6
+ ; REGALLOC-NEXT: SI_SPILL_V32_SAVE killed $vgpr0, %stack.3, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.3, addrspace 5)
; REGALLOC-NEXT: renamable $sgpr6_sgpr7 = COPY $exec, implicit-def $exec
; REGALLOC-NEXT: renamable $sgpr4_sgpr5 = S_AND_B64 renamable $sgpr6_sgpr7, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
; REGALLOC-NEXT: renamable $sgpr6_sgpr7 = S_XOR_B64 renamable $sgpr4_sgpr5, killed renamable $sgpr6_sgpr7, implicit-def dead $scc
@@ -31,8 +33,8 @@ define i32 @prolog_spill(i32 %arg0, i32 %arg1, i32 %arg2) {
; REGALLOC-NEXT: $sgpr4 = SI_RESTORE_S32_FROM_VGPR $vgpr63, 0, implicit-def $sgpr4_sgpr5
; REGALLOC-NEXT: $sgpr5 = SI_RESTORE_S32_FROM_VGPR $vgpr63, 1
; REGALLOC-NEXT: renamable $sgpr4_sgpr5 = S_OR_SAVEEXEC_B64 killed renamable $sgpr4_sgpr5, implicit-def $exec, implicit-def dead $scc, implicit $exec
- ; REGALLOC-NEXT: $vgpr0 = SI_SPILL_V32_RESTORE %stack.6, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.6, addrspace 5)
- ; REGALLOC-NEXT: SI_SPILL_V32_SAVE killed $vgpr0, %stack.5, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.5, addrspace 5)
+ ; REGALLOC-NEXT: $vgpr0 = SI_SPILL_V32_RESTORE %stack.3, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.3, addrspace 5)
+ ; REGALLOC-NEXT: SI_SPILL_V32_SAVE killed $vgpr0, %stack.6, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.6, addrspace 5)
; REGALLOC-NEXT: renamable $sgpr4_sgpr5 = S_AND_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
; REGALLOC-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR killed $sgpr4, 2, $vgpr63, implicit-def $sgpr4_sgpr5, implicit $sgpr4_sgpr5
; REGALLOC-NEXT: $vgpr63 = SI_SPILL_S32_TO_VGPR $sgpr5, 3, $vgpr63, implicit $sgpr4_sgpr5
@@ -44,19 +46,19 @@ define i32 @prolog_spill(i32 %arg0, i32 %arg1, i32 %arg2) {
; REGALLOC-NEXT: bb.2.bb.1:
; REGALLOC-NEXT: successors: %bb.4(0x80000000)
; REGALLOC-NEXT: {{ $}}
- ; REGALLOC-NEXT: $vgpr0 = SI_SPILL_V32_RESTORE %stack.3, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.3, addrspace 5)
+ ; REGALLOC-NEXT: $vgpr0 = SI_SPILL_V32_RESTORE %stack.4, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.4, addrspace 5)
; REGALLOC-NEXT: renamable $sgpr4 = S_MOV_B32 10
; REGALLOC-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr0, killed $sgpr4, 0, implicit $exec
- ; REGALLOC-NEXT: SI_SPILL_V32_SAVE killed $vgpr0, %stack.5, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.5, addrspace 5)
+ ; REGALLOC-NEXT: SI_SPILL_V32_SAVE killed $vgpr0, %stack.6, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.6, addrspace 5)
; REGALLOC-NEXT: S_BRANCH %bb.4
; REGALLOC-NEXT: {{ $}}
; REGALLOC-NEXT: bb.3.bb.2:
; REGALLOC-NEXT: successors: %bb.1(0x80000000)
; REGALLOC-NEXT: {{ $}}
- ; REGALLOC-NEXT: $vgpr0 = SI_SPILL_V32_RESTORE %stack.4, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.4, addrspace 5)
+ ; REGALLOC-NEXT: $vgpr0 = SI_SPILL_V32_RESTORE %stack.5, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.5, addrspace 5)
; REGALLOC-NEXT: renamable $sgpr4 = S_MOV_B32 20
; REGALLOC-NEXT: renamable $vgpr0 = V_ADD_U32_e64 $vgpr0, killed $sgpr4, 0, implicit $exec
- ; REGALLOC-NEXT: SI_SPILL_V32_SAVE killed $vgpr0, %stack.6, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.6, addrspace 5)
+ ; REGALLOC-NEXT: SI_SPILL_V32_SAVE killed $vgpr0, %stack.3, $sgpr32, 0, implicit $exec :: (store (s32) into %stack.3, addrspace 5)
; REGALLOC-NEXT: S_BRANCH %bb.1
; REGALLOC-NEXT: {{ $}}
; REGALLOC-NEXT: bb.4.bb.3:
@@ -64,7 +66,7 @@ define i32 @prolog_spill(i32 %arg0, i32 %arg1, i32 %arg2) {
; REGALLOC-NEXT: $sgpr4 = SI_RESTORE_S32_FROM_VGPR $vgpr63, 2, implicit-def $sgpr4_sgpr5
; REGALLOC-NEXT: $sgpr5 = SI_RESTORE_S32_FROM_VGPR killed $vgpr63, 3
; REGALLOC-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
- ; REGALLOC-NEXT: $vgpr0 = SI_SPILL_V32_RESTORE %stack.5, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.5, addrspace 5)
+ ; REGALLOC-NEXT: $vgpr0 = SI_SPILL_V32_RESTORE %stack.6, $sgpr32, 0, implicit $exec :: (load (s32) from %stack.6, addrspace 5)
; REGALLOC-NEXT: renamable $vgpr0 = V_LSHL_ADD_U32_e64 killed $vgpr0, 2, $vgpr0, implicit $exec
; REGALLOC-NEXT: SI_RETURN implicit killed $vgpr0
bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-flat.ll b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-flat.ll
index 891e1fa661866..3577a2ed636bf 100644
--- a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-flat.ll
+++ b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-flat.ll
@@ -73,76 +73,76 @@ define void @test_sinkable_flat_small_offset_i32(ptr %out, ptr %in, i32 %cond) {
; GFX7-LABEL: test_sinkable_flat_small_offset_i32:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v5, 0
; GFX7-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX7-NEXT: v_mov_b32_e32 v4, 0
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7-NEXT: s_cbranch_execz .LBB0_2
; GFX7-NEXT: ; %bb.1: ; %if
; GFX7-NEXT: v_add_i32_e32 v2, vcc, 28, v2
; GFX7-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GFX7-NEXT: flat_load_dword v5, v[2:3]
+; GFX7-NEXT: flat_load_dword v4, v[2:3]
; GFX7-NEXT: .LBB0_2: ; %endif
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x3d08fc, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: flat_store_dword v[0:1], v5
+; GFX7-NEXT: flat_store_dword v[0:1], v4
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: test_sinkable_flat_small_offset_i32:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v5, 0
; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX8-NEXT: v_mov_b32_e32 v4, 0
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB0_2
; GFX8-NEXT: ; %bb.1: ; %if
; GFX8-NEXT: v_add_u32_e32 v2, vcc, 28, v2
; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GFX8-NEXT: flat_load_dword v5, v[2:3]
+; GFX8-NEXT: flat_load_dword v4, v[2:3]
; GFX8-NEXT: .LBB0_2: ; %endif
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x3d08fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_store_dword v[0:1], v5
+; GFX8-NEXT: flat_store_dword v[0:1], v4
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_sinkable_flat_small_offset_i32:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v5, 0
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB0_2
; GFX9-NEXT: ; %bb.1: ; %if
-; GFX9-NEXT: flat_load_dword v5, v[2:3] offset:28
+; GFX9-NEXT: flat_load_dword v4, v[2:3] offset:28
; GFX9-NEXT: .LBB0_2: ; %endif
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 0x3d0000, v0
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: flat_store_dword v[0:1], v5 offset:2300
+; GFX9-NEXT: flat_store_dword v[0:1], v4 offset:2300
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: test_sinkable_flat_small_offset_i32:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: s_mov_b32 s4, exec_lo
-; GFX10-NEXT: v_cmpx_ne_u32_e32 0, v4
+; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4
+; GFX10-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB0_2
; GFX10-NEXT: ; %bb.1: ; %if
-; GFX10-NEXT: flat_load_dword v5, v[2:3] offset:28
+; GFX10-NEXT: flat_load_dword v4, v[2:3] offset:28
; GFX10-NEXT: .LBB0_2: ; %endif
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x3d0800, v0
; GFX10-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: flat_store_dword v[0:1], v5 offset:252
+; GFX10-NEXT: flat_store_dword v[0:1], v4 offset:252
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -228,78 +228,78 @@ define void @test_sink_noop_addrspacecast_flat_to_global_i32(ptr %out, ptr %in,
; GFX7-LABEL: test_sink_noop_addrspacecast_flat_to_global_i32:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v5, 0
+; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX7-NEXT: v_mov_b32_e32 v4, 0
+; GFX7-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX7-NEXT: s_cbranch_execz .LBB1_2
; GFX7-NEXT: ; %bb.1: ; %if
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, 0xf000
-; GFX7-NEXT: s_mov_b32 s8, s10
-; GFX7-NEXT: s_mov_b32 s9, s10
-; GFX7-NEXT: buffer_load_dword v5, v[2:3], s[8:11], 0 addr64 offset:28
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s4, s6
+; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: buffer_load_dword v4, v[2:3], s[4:7], 0 addr64 offset:28
; GFX7-NEXT: .LBB1_2: ; %endif
-; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x3d08fc, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: flat_store_dword v[0:1], v5
+; GFX7-NEXT: flat_store_dword v[0:1], v4
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: test_sink_noop_addrspacecast_flat_to_global_i32:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v5, 0
; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX8-NEXT: v_mov_b32_e32 v4, 0
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB1_2
; GFX8-NEXT: ; %bb.1: ; %if
; GFX8-NEXT: v_add_u32_e32 v2, vcc, 28, v2
; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GFX8-NEXT: flat_load_dword v5, v[2:3]
+; GFX8-NEXT: flat_load_dword v4, v[2:3]
; GFX8-NEXT: .LBB1_2: ; %endif
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x3d08fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: flat_store_dword v[0:1], v5
+; GFX8-NEXT: flat_store_dword v[0:1], v4
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_sink_noop_addrspacecast_flat_to_global_i32:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v5, 0
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB1_2
; GFX9-NEXT: ; %bb.1: ; %if
-; GFX9-NEXT: global_load_dword v5, v[2:3], off offset:28
+; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:28
; GFX9-NEXT: .LBB1_2: ; %endif
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 0x3d0000, v0
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: flat_store_dword v[0:1], v5 offset:2300
+; GFX9-NEXT: flat_store_dword v[0:1], v4 offset:2300
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: test_sink_noop_addrspacecast_flat_to_global_i32:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: s_mov_b32 s4, exec_lo
-; GFX10-NEXT: v_cmpx_ne_u32_e32 0, v4
+; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4
+; GFX10-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB1_2
; GFX10-NEXT: ; %bb.1: ; %if
-; GFX10-NEXT: global_load_dword v5, v[2:3], off offset:28
+; GFX10-NEXT: global_load_dword v4, v[2:3], off offset:28
; GFX10-NEXT: .LBB1_2: ; %endif
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x3d0800, v0
; GFX10-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: flat_store_dword v[0:1], v5 offset:252
+; GFX10-NEXT: flat_store_dword v[0:1], v4 offset:252
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -341,78 +341,78 @@ define void @test_sink_noop_addrspacecast_flat_to_constant_i32(ptr %out, ptr %in
; GFX7-LABEL: test_sink_noop_addrspacecast_flat_to_constant_i32:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v5, 0
+; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX7-NEXT: v_mov_b32_e32 v4, 0
+; GFX7-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GFX7-NEXT: s_cbranch_execz .LBB2_2
; GFX7-NEXT: ; %bb.1: ; %if
-; GFX7-NEXT: s_mov_b32 s10, 0
-; GFX7-NEXT: s_mov_b32 s11, 0xf000
-; GFX7-NEXT: s_mov_b32 s8, s10
-; GFX7-NEXT: s_mov_b32 s9, s10
-; GFX7-NEXT: buffer_load_dword v5, v[2:3], s[8:11], 0 addr64 offset:28
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s4, s6
+; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: buffer_load_dword v4, v[2:3], s[4:7], 0 addr64 offset:28
; GFX7-NEXT: .LBB2_2: ; %endif
-; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x3d08fc, v0
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: flat_store_dword v[0:1], v5
+; GFX7-NEXT: flat_store_dword v[0:1], v4
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: test_sink_noop_addrspacecast_flat_to_constant_i32:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v5, 0
; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX8-NEXT: v_mov_b32_e32 v4, 0
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB2_2
; GFX8-NEXT: ; %bb.1: ; %if
; GFX8-NEXT: v_add_u32_e32 v2, vcc, 28, v2
; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GFX8-NEXT: flat_load_dword v5, v[2:3]
+; GFX8-NEXT: flat_load_dword v4, v[2:3]
; GFX8-NEXT: .LBB2_2: ; %endif
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x3d08fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: flat_store_dword v[0:1], v5
+; GFX8-NEXT: flat_store_dword v[0:1], v4
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_sink_noop_addrspacecast_flat_to_constant_i32:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v5, 0
; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB2_2
; GFX9-NEXT: ; %bb.1: ; %if
-; GFX9-NEXT: global_load_dword v5, v[2:3], off offset:28
+; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:28
; GFX9-NEXT: .LBB2_2: ; %endif
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 0x3d0000, v0
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: flat_store_dword v[0:1], v5 offset:2300
+; GFX9-NEXT: flat_store_dword v[0:1], v4 offset:2300
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: test_sink_noop_addrspacecast_flat_to_constant_i32:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: s_mov_b32 s4, exec_lo
-; GFX10-NEXT: v_cmpx_ne_u32_e32 0, v4
+; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4
+; GFX10-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB2_2
; GFX10-NEXT: ; %bb.1: ; %if
-; GFX10-NEXT: global_load_dword v5, v[2:3], off offset:28
+; GFX10-NEXT: global_load_dword v4, v[2:3], off offset:28
; GFX10-NEXT: .LBB2_2: ; %endif
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x3d0800, v0
; GFX10-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: flat_store_dword v[0:1], v5 offset:252
+; GFX10-NEXT: flat_store_dword v[0:1], v4 offset:252
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -509,9 +509,9 @@ define void @test_sink_flat_small_max_flat_offset(ptr %out, ptr %in) #1 {
; GFX7-LABEL: test_sink_flat_small_max_flat_offset:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v5, -1, 0
+; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v4, -1, 0
+; GFX7-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
; GFX7-NEXT: v_mov_b32_e32 v4, 0
-; GFX7-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7-NEXT: s_cbranch_execz .LBB3_2
; GFX7-NEXT: ; %bb.1: ; %if
@@ -530,9 +530,9 @@ define void @test_sink_flat_small_max_flat_offset(ptr %out, ptr %in) #1 {
; GFX8-LABEL: test_sink_flat_small_max_flat_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
; GFX8-NEXT: v_mov_b32_e32 v4, 0
-; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB3_2
; GFX8-NEXT: ; %bb.1: ; %if
@@ -551,9 +551,9 @@ define void @test_sink_flat_small_max_flat_offset(ptr %out, ptr %in) #1 {
; GFX9-LABEL: test_sink_flat_small_max_flat_offset:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB3_2
; GFX9-NEXT: ; %bb.1: ; %if
@@ -570,10 +570,10 @@ define void @test_sink_flat_small_max_flat_offset(ptr %out, ptr %in) #1 {
; GFX10-LABEL: test_sink_flat_small_max_flat_offset:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4
; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: s_mov_b32 s4, exec_lo
-; GFX10-NEXT: v_cmpx_ne_u32_e32 0, v5
+; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB3_2
; GFX10-NEXT: ; %bb.1: ; %if
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, 0x800, v2
@@ -630,9 +630,9 @@ define void @test_sink_flat_small_max_plus_1_flat_offset(ptr %out, ptr %in) #1 {
; GFX7-LABEL: test_sink_flat_small_max_plus_1_flat_offset:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v5, -1, 0
+; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v4, -1, 0
+; GFX7-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
; GFX7-NEXT: v_mov_b32_e32 v4, 0
-; GFX7-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7-NEXT: s_cbranch_execz .LBB4_2
; GFX7-NEXT: ; %bb.1: ; %if
@@ -651,9 +651,9 @@ define void @test_sink_flat_small_max_plus_1_flat_offset(ptr %out, ptr %in) #1 {
; GFX8-LABEL: test_sink_flat_small_max_plus_1_flat_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
; GFX8-NEXT: v_mov_b32_e32 v4, 0
-; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB4_2
; GFX8-NEXT: ; %bb.1: ; %if
@@ -672,9 +672,9 @@ define void @test_sink_flat_small_max_plus_1_flat_offset(ptr %out, ptr %in) #1 {
; GFX9-LABEL: test_sink_flat_small_max_plus_1_flat_offset:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB4_2
; GFX9-NEXT: ; %bb.1: ; %if
@@ -693,10 +693,10 @@ define void @test_sink_flat_small_max_plus_1_flat_offset(ptr %out, ptr %in) #1 {
; GFX10-LABEL: test_sink_flat_small_max_plus_1_flat_offset:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v5, -1, 0
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v4, -1, 0
+; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4
; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: s_mov_b32 s4, exec_lo
-; GFX10-NEXT: v_cmpx_ne_u32_e32 0, v5
+; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB4_2
; GFX10-NEXT: ; %bb.1: ; %if
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, 0x1000, v2
@@ -753,9 +753,9 @@ define void @test_sinkable_flat_reg_offset(ptr %out, ptr %in, i64 %reg) #1 {
; GFX7-LABEL: test_sinkable_flat_reg_offset:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v7, -1, 0
+; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v6, -1, 0
+; GFX7-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6
; GFX7-NEXT: v_mov_b32_e32 v6, 0
-; GFX7-NEXT: v_cmp_ne_u32_e32 vcc, 0, v7
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7-NEXT: s_cbranch_execz .LBB5_2
; GFX7-NEXT: ; %bb.1: ; %if
@@ -774,9 +774,9 @@ define void @test_sinkable_flat_reg_offset(ptr %out, ptr %in, i64 %reg) #1 {
; GFX8-LABEL: test_sinkable_flat_reg_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v6, -1, 0
+; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6
; GFX8-NEXT: v_mov_b32_e32 v6, 0
-; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v7
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB5_2
; GFX8-NEXT: ; %bb.1: ; %if
@@ -795,9 +795,9 @@ define void @test_sinkable_flat_reg_offset(ptr %out, ptr %in, i64 %reg) #1 {
; GFX9-LABEL: test_sinkable_flat_reg_offset:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v6, -1, 0
+; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6
; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v7
; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB5_2
; GFX9-NEXT: ; %bb.1: ; %if
@@ -816,10 +816,10 @@ define void @test_sinkable_flat_reg_offset(ptr %out, ptr %in, i64 %reg) #1 {
; GFX10-LABEL: test_sinkable_flat_reg_offset:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v7, -1, 0
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v6, -1, 0
+; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v6
; GFX10-NEXT: v_mov_b32_e32 v6, 0
-; GFX10-NEXT: s_mov_b32 s4, exec_lo
-; GFX10-NEXT: v_cmpx_ne_u32_e32 0, v7
+; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB5_2
; GFX10-NEXT: ; %bb.1: ; %if
; GFX10-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
diff --git a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll
index d448dc6bb78b6..ca11876b461ee 100644
--- a/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll
+++ b/llvm/test/CodeGen/AMDGPU/cgp-addressing-modes-gfx1030.ll
@@ -25,10 +25,10 @@ define amdgpu_kernel void @test_sink_small_offset_global_atomic_csub_i32(ptr add
; GCN-LABEL: test_sink_small_offset_global_atomic_csub_i32:
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; GCN-NEXT: v_mbcnt_lo_u32_b32 v1, -1, 0
+; GCN-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0
+; GCN-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: s_mov_b32 s4, exec_lo
-; GCN-NEXT: v_cmpx_ne_u32_e32 0, v1
+; GCN-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GCN-NEXT: s_cbranch_execz .LBB0_2
; GCN-NEXT: ; %bb.1: ; %if
; GCN-NEXT: v_mov_b32_e32 v0, 0
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
index 97a0d6880c6b6..5ce8ca04603ca 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
+++ b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
@@ -11,30 +11,29 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; COEXEC-NEXT: v_mov_b32_e32 v0, 0
-; COEXEC-NEXT: s_clause 0x1
; COEXEC-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
+; COEXEC-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
+; COEXEC-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v4, 0
+; COEXEC-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_mov_b32 v6, 0
+; COEXEC-NEXT: v_dual_mov_b32 v7, 0 :: v_dual_mov_b32 v8, 0
+; COEXEC-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v10, 0
+; COEXEC-NEXT: v_mov_b32_e32 v11, 0
; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv
-; COEXEC-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
-; COEXEC-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0
-; COEXEC-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0
-; COEXEC-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, v0
-; COEXEC-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0
-; COEXEC-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0
-; COEXEC-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0
-; COEXEC-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v16, v0
-; COEXEC-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v0
-; COEXEC-NEXT: v_dual_mov_b32 v19, v0 :: v_dual_mov_b32 v20, v0
-; COEXEC-NEXT: v_dual_mov_b32 v21, v0 :: v_dual_mov_b32 v22, v0
-; COEXEC-NEXT: v_dual_mov_b32 v23, v0 :: v_dual_mov_b32 v24, v0
-; COEXEC-NEXT: v_dual_mov_b32 v25, v0 :: v_dual_mov_b32 v26, v0
-; COEXEC-NEXT: v_dual_mov_b32 v27, v0 :: v_dual_mov_b32 v28, v0
+; COEXEC-NEXT: v_dual_mov_b32 v12, 0 :: v_dual_mov_b32 v13, 0
+; COEXEC-NEXT: v_dual_mov_b32 v14, 0 :: v_dual_mov_b32 v15, 0
+; COEXEC-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v17, 0
+; COEXEC-NEXT: v_dual_mov_b32 v18, 0 :: v_dual_mov_b32 v19, 0
+; COEXEC-NEXT: v_dual_mov_b32 v20, 0 :: v_dual_mov_b32 v21, 0
+; COEXEC-NEXT: v_dual_mov_b32 v22, 0 :: v_dual_mov_b32 v23, 0
+; COEXEC-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v25, 0
+; COEXEC-NEXT: v_dual_mov_b32 v26, 0 :: v_dual_mov_b32 v27, 0
+; COEXEC-NEXT: v_dual_mov_b32 v28, 0 :: v_dual_mov_b32 v29, 0
+; COEXEC-NEXT: v_mov_b32_e32 v30, 0
; COEXEC-NEXT: s_wait_kmcnt 0x0
; COEXEC-NEXT: s_bitcmp1_b32 s0, 0
-; COEXEC-NEXT: v_mov_b32_e32 v29, v0
+; COEXEC-NEXT: v_mov_b32_e32 v31, 0
; COEXEC-NEXT: s_cselect_b32 s0, -1, 0
-; COEXEC-NEXT: v_mov_b32_e32 v30, v0
; COEXEC-NEXT: s_xor_b32 s0, s0, -1
-; COEXEC-NEXT: v_mov_b32_e32 v31, v0
; COEXEC-NEXT: .LBB0_1: ; %loop
; COEXEC-NEXT: ; =>This Inner Loop Header: Depth=1
; COEXEC-NEXT: v_nop
@@ -99,27 +98,26 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; GCN-NEXT: v_nop
; GCN-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GCN-NEXT: s_clause 0x1
-; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv
; GCN-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
-; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
-; GCN-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0
-; GCN-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0
-; GCN-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, v0
-; GCN-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0
-; GCN-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0
-; GCN-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0
+; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv
+; GCN-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GCN-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
+; GCN-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0
+; GCN-NEXT: v_dual_mov_b32 v6, 0 :: v_dual_mov_b32 v7, 0
+; GCN-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v9, 0
+; GCN-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v11, 0
+; GCN-NEXT: v_dual_mov_b32 v12, 0 :: v_dual_mov_b32 v13, 0
+; GCN-NEXT: v_dual_mov_b32 v14, 0 :: v_dual_mov_b32 v15, 0
+; GCN-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v17, 0
+; GCN-NEXT: v_dual_mov_b32 v18, 0 :: v_dual_mov_b32 v19, 0
; GCN-NEXT: s_wait_kmcnt 0x0
; GCN-NEXT: s_bitcmp1_b32 s0, 0
-; GCN-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v16, v0
-; GCN-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v0
-; GCN-NEXT: v_dual_mov_b32 v19, v0 :: v_dual_mov_b32 v20, v0
-; GCN-NEXT: v_dual_mov_b32 v21, v0 :: v_dual_mov_b32 v22, v0
-; GCN-NEXT: v_dual_mov_b32 v23, v0 :: v_dual_mov_b32 v24, v0
-; GCN-NEXT: v_dual_mov_b32 v25, v0 :: v_dual_mov_b32 v26, v0
-; GCN-NEXT: v_dual_mov_b32 v27, v0 :: v_dual_mov_b32 v28, v0
-; GCN-NEXT: v_dual_mov_b32 v29, v0 :: v_dual_mov_b32 v30, v0
-; GCN-NEXT: v_mov_b32_e32 v31, v0
+; GCN-NEXT: v_dual_mov_b32 v20, 0 :: v_dual_mov_b32 v21, 0
+; GCN-NEXT: v_dual_mov_b32 v22, 0 :: v_dual_mov_b32 v23, 0
+; GCN-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v25, 0
+; GCN-NEXT: v_dual_mov_b32 v26, 0 :: v_dual_mov_b32 v27, 0
+; GCN-NEXT: v_dual_mov_b32 v28, 0 :: v_dual_mov_b32 v29, 0
+; GCN-NEXT: v_dual_mov_b32 v30, 0 :: v_dual_mov_b32 v31, 0
; GCN-NEXT: s_cselect_b32 s0, -1, 0
; GCN-NEXT: s_xor_b32 s0, s0, -1
; GCN-NEXT: .LBB0_1: ; %loop
@@ -257,32 +255,32 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; COEXEC-NEXT: s_mov_b64 s[64:65], 0
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; COEXEC-NEXT: s_mov_b32 s6, 0
-; COEXEC-NEXT: s_clause 0x1
+; COEXEC-NEXT: v_mov_b32_e32 v0, 0
; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; COEXEC-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
+; COEXEC-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v4, 0
+; COEXEC-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_mov_b32 v6, 0
+; COEXEC-NEXT: v_dual_mov_b32 v7, 0 :: v_dual_mov_b32 v8, 0
+; COEXEC-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v10, 0
+; COEXEC-NEXT: v_mov_b32_e32 v11, 0
; COEXEC-NEXT: s_load_b64 s[2:3], s[4:5], 0x10 nv
-; COEXEC-NEXT: v_mov_b32_e32 v0, 0
-; COEXEC-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
-; COEXEC-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0
-; COEXEC-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0
-; COEXEC-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, v0
-; COEXEC-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0
-; COEXEC-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0
-; COEXEC-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0
-; COEXEC-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v16, v0
-; COEXEC-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v0
-; COEXEC-NEXT: v_dual_mov_b32 v19, v0 :: v_dual_mov_b32 v20, v0
-; COEXEC-NEXT: v_dual_mov_b32 v21, v0 :: v_dual_mov_b32 v22, v0
-; COEXEC-NEXT: v_dual_mov_b32 v23, v0 :: v_dual_mov_b32 v24, v0
-; COEXEC-NEXT: v_dual_mov_b32 v25, v0 :: v_dual_mov_b32 v26, v0
-; COEXEC-NEXT: v_dual_mov_b32 v27, v0 :: v_dual_mov_b32 v28, v0
+; COEXEC-NEXT: v_dual_mov_b32 v12, 0 :: v_dual_mov_b32 v13, 0
+; COEXEC-NEXT: v_dual_mov_b32 v14, 0 :: v_dual_mov_b32 v15, 0
+; COEXEC-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v17, 0
+; COEXEC-NEXT: v_dual_mov_b32 v18, 0 :: v_dual_mov_b32 v19, 0
+; COEXEC-NEXT: v_dual_mov_b32 v20, 0 :: v_dual_mov_b32 v21, 0
+; COEXEC-NEXT: v_dual_mov_b32 v22, 0 :: v_dual_mov_b32 v23, 0
+; COEXEC-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v25, 0
+; COEXEC-NEXT: v_dual_mov_b32 v26, 0 :: v_dual_mov_b32 v27, 0
+; COEXEC-NEXT: v_mov_b32_e32 v28, 0
+; COEXEC-NEXT: s_mov_b32 s6, 0
+; COEXEC-NEXT: v_mov_b32_e32 v29, 0
; COEXEC-NEXT: s_wait_kmcnt 0x0
; COEXEC-NEXT: s_bitcmp1_b32 s2, 0
-; COEXEC-NEXT: v_mov_b32_e32 v29, v0
+; COEXEC-NEXT: v_mov_b32_e32 v30, 0
; COEXEC-NEXT: s_cselect_b32 s2, -1, 0
-; COEXEC-NEXT: v_mov_b32_e32 v30, v0
+; COEXEC-NEXT: v_mov_b32_e32 v31, 0
; COEXEC-NEXT: s_xor_b32 s2, s2, -1
-; COEXEC-NEXT: v_mov_b32_e32 v31, v0
; COEXEC-NEXT: .LBB1_1: ; %loop
; COEXEC-NEXT: ; =>This Inner Loop Header: Depth=1
; COEXEC-NEXT: s_add_co_i32 s7, s0, s6
@@ -377,37 +375,36 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; GCN-NEXT: v_nop
; GCN-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GCN-NEXT: s_clause 0x1
-; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv
-; GCN-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv
-; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: s_mov_b32 s6, 0
-; GCN-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
-; GCN-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0
-; GCN-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0
-; GCN-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, v0
-; GCN-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0
-; GCN-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0
-; GCN-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0
+; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GCN-NEXT: s_load_b64 s[2:3], s[4:5], 0x10 nv
+; GCN-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GCN-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
+; GCN-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0
+; GCN-NEXT: v_dual_mov_b32 v6, 0 :: v_dual_mov_b32 v7, 0
+; GCN-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v9, 0
+; GCN-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v11, 0
+; GCN-NEXT: v_dual_mov_b32 v12, 0 :: v_dual_mov_b32 v13, 0
+; GCN-NEXT: v_dual_mov_b32 v14, 0 :: v_dual_mov_b32 v15, 0
+; GCN-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v17, 0
+; GCN-NEXT: v_dual_mov_b32 v18, 0 :: v_dual_mov_b32 v19, 0
; GCN-NEXT: s_wait_kmcnt 0x0
-; GCN-NEXT: s_bitcmp1_b32 s0, 0
-; GCN-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v16, v0
-; GCN-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v0
-; GCN-NEXT: v_dual_mov_b32 v19, v0 :: v_dual_mov_b32 v20, v0
-; GCN-NEXT: v_dual_mov_b32 v21, v0 :: v_dual_mov_b32 v22, v0
-; GCN-NEXT: v_dual_mov_b32 v23, v0 :: v_dual_mov_b32 v24, v0
-; GCN-NEXT: v_dual_mov_b32 v25, v0 :: v_dual_mov_b32 v26, v0
-; GCN-NEXT: v_dual_mov_b32 v27, v0 :: v_dual_mov_b32 v28, v0
-; GCN-NEXT: v_dual_mov_b32 v29, v0 :: v_dual_mov_b32 v30, v0
-; GCN-NEXT: v_mov_b32_e32 v31, v0
-; GCN-NEXT: s_cselect_b32 s0, -1, 0
-; GCN-NEXT: s_xor_b32 s0, s0, -1
+; GCN-NEXT: s_bitcmp1_b32 s2, 0
+; GCN-NEXT: v_dual_mov_b32 v20, 0 :: v_dual_mov_b32 v21, 0
+; GCN-NEXT: v_dual_mov_b32 v22, 0 :: v_dual_mov_b32 v23, 0
+; GCN-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v25, 0
+; GCN-NEXT: v_dual_mov_b32 v26, 0 :: v_dual_mov_b32 v27, 0
+; GCN-NEXT: v_dual_mov_b32 v28, 0 :: v_dual_mov_b32 v29, 0
+; GCN-NEXT: v_dual_mov_b32 v30, 0 :: v_dual_mov_b32 v31, 0
+; GCN-NEXT: s_cselect_b32 s2, -1, 0
+; GCN-NEXT: s_mov_b32 s6, 0
+; GCN-NEXT: s_xor_b32 s2, s2, -1
; GCN-NEXT: .LBB1_1: ; %loop
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT: s_add_co_i32 s7, s2, s6
-; GCN-NEXT: s_add_co_i32 s8, s3, s6
+; GCN-NEXT: s_add_co_i32 s7, s0, s6
+; GCN-NEXT: s_add_co_i32 s8, s1, s6
; GCN-NEXT: v_dual_mov_b32 v96, s7 :: v_dual_mov_b32 v97, s8
-; GCN-NEXT: s_add_co_i32 s6, s6, s1
-; GCN-NEXT: s_and_b32 s7, s0, exec_lo
+; GCN-NEXT: s_add_co_i32 s6, s6, s3
+; GCN-NEXT: s_and_b32 s7, s2, exec_lo
; GCN-NEXT: s_cselect_b32 s7, 1, 0
; GCN-NEXT: ds_load_tr16_b128 v[32:35], v96
; GCN-NEXT: ds_load_tr16_b128 v[36:39], v96 offset:64
@@ -612,48 +609,49 @@ define amdgpu_kernel void @ds_wmma_block_carried(ptr addrspace(3) %base, ptr add
; COEXEC-NEXT: s_mov_b64 s[64:65], 0
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; COEXEC-NEXT: v_mov_b32_e32 v0, 0
+; COEXEC-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
+; COEXEC-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
+; COEXEC-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v4, 0
+; COEXEC-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_mov_b32 v6, 0
+; COEXEC-NEXT: v_dual_mov_b32 v7, 0 :: v_dual_mov_b32 v8, 0
+; COEXEC-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v10, 0
+; COEXEC-NEXT: v_mov_b32_e32 v11, 0
+; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv
+; COEXEC-NEXT: v_dual_mov_b32 v12, 0 :: v_dual_mov_b32 v13, 0
+; COEXEC-NEXT: v_dual_mov_b32 v14, 0 :: v_dual_mov_b32 v15, 0
+; COEXEC-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v17, 0
+; COEXEC-NEXT: v_dual_mov_b32 v18, 0 :: v_dual_mov_b32 v19, 0
+; COEXEC-NEXT: v_dual_mov_b32 v20, 0 :: v_dual_mov_b32 v21, 0
+; COEXEC-NEXT: v_dual_mov_b32 v22, 0 :: v_dual_mov_b32 v23, 0
+; COEXEC-NEXT: s_wait_kmcnt 0x0
+; COEXEC-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v28, s2
+; COEXEC-NEXT: v_dual_mov_b32 v25, 0 :: v_dual_mov_b32 v26, 0
+; COEXEC-NEXT: v_mov_b32_e32 v27, 0
; COEXEC-NEXT: s_mov_b32 s8, 0
-; COEXEC-NEXT: s_load_b32 s3, s[4:5], 0x0 nv
+; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v28
+; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v28 offset:64
+; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v28 offset:128
+; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v28 offset:192
+; COEXEC-NEXT: ds_load_tr16_b128 v[48:51], v28 offset:256
+; COEXEC-NEXT: ds_load_tr16_b128 v[52:55], v28 offset:320
+; COEXEC-NEXT: ds_load_tr16_b128 v[56:59], v28 offset:384
+; COEXEC-NEXT: ds_load_tr16_b128 v[60:63], v28 offset:448
; COEXEC-NEXT: s_mov_b32 s9, s8
; COEXEC-NEXT: s_mov_b32 s10, s8
; COEXEC-NEXT: s_mov_b32 s11, s8
+; COEXEC-NEXT: v_mov_b32_e32 v28, 0
+; COEXEC-NEXT: s_bitcmp1_b32 s0, 0
; COEXEC-NEXT: s_mov_b32 s12, s8
+; COEXEC-NEXT: v_mov_b32_e32 v29, 0
+; COEXEC-NEXT: s_cselect_b32 s3, -1, 0
; COEXEC-NEXT: s_mov_b32 s13, s8
+; COEXEC-NEXT: v_mov_b32_e32 v30, 0
+; COEXEC-NEXT: s_add_co_i32 s0, s2, s1
; COEXEC-NEXT: s_mov_b32 s14, s8
+; COEXEC-NEXT: v_mov_b32_e32 v31, 0
+; COEXEC-NEXT: s_xor_b32 s2, s3, -1
; COEXEC-NEXT: s_mov_b32 s15, s8
-; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv
-; COEXEC-NEXT: v_mov_b32_e32 v0, 0
-; COEXEC-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
-; COEXEC-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0
-; COEXEC-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0
-; COEXEC-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, v0
-; COEXEC-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0
-; COEXEC-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0
-; COEXEC-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0
-; COEXEC-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v16, v0
-; COEXEC-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v0
-; COEXEC-NEXT: v_dual_mov_b32 v19, v0 :: v_dual_mov_b32 v20, v0
-; COEXEC-NEXT: v_dual_mov_b32 v21, v0 :: v_dual_mov_b32 v22, v0
-; COEXEC-NEXT: v_dual_mov_b32 v23, v0 :: v_dual_mov_b32 v24, v0
-; COEXEC-NEXT: v_dual_mov_b32 v25, v0 :: v_dual_mov_b32 v26, v0
-; COEXEC-NEXT: v_dual_mov_b32 v27, v0 :: v_dual_mov_b32 v28, v0
-; COEXEC-NEXT: s_wait_kmcnt 0x0
-; COEXEC-NEXT: s_bitcmp1_b32 s0, 0
-; COEXEC-NEXT: v_mov_b32_e32 v29, v0
-; COEXEC-NEXT: s_cselect_b32 s2, -1, 0
-; COEXEC-NEXT: v_mov_b32_e32 v30, v0
-; COEXEC-NEXT: s_add_co_i32 s0, s3, s1
-; COEXEC-NEXT: v_mov_b32_e32 v31, v0
-; COEXEC-NEXT: s_xor_b32 s2, s2, -1
-; COEXEC-NEXT: v_mov_b32_e32 v60, s3
-; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v60
-; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v60 offset:64
-; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v60 offset:128
-; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v60 offset:192
-; COEXEC-NEXT: ds_load_tr16_b128 v[48:51], v60 offset:256
-; COEXEC-NEXT: ds_load_tr16_b128 v[52:55], v60 offset:320
-; COEXEC-NEXT: ds_load_tr16_b128 v[56:59], v60 offset:384
-; COEXEC-NEXT: ds_load_tr16_b128 v[60:63], v60 offset:448
; COEXEC-NEXT: s_wait_dscnt 0x0
; COEXEC-NEXT: .LBB2_1: ; %loop
; COEXEC-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -715,8 +713,37 @@ define amdgpu_kernel void @ds_wmma_block_carried(ptr addrspace(3) %base, ptr add
; GCN-NEXT: s_clause 0x1
; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv
; GCN-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
-; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v0, 0
+; GCN-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 0
+; GCN-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v4, 0
+; GCN-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_mov_b32 v6, 0
+; GCN-NEXT: v_dual_mov_b32 v7, 0 :: v_dual_mov_b32 v8, 0
+; GCN-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v11, 0
+; GCN-NEXT: v_dual_mov_b32 v13, 0 :: v_dual_mov_b32 v14, 0
+; GCN-NEXT: v_dual_mov_b32 v15, 0 :: v_dual_mov_b32 v16, 0
+; GCN-NEXT: v_dual_mov_b32 v17, 0 :: v_dual_mov_b32 v18, 0
+; GCN-NEXT: s_wait_kmcnt 0x0
+; GCN-NEXT: s_bitcmp1_b32 s0, 0
+; GCN-NEXT: v_dual_mov_b32 v12, s2 :: v_dual_mov_b32 v19, 0
+; GCN-NEXT: v_dual_mov_b32 v20, 0 :: v_dual_mov_b32 v21, 0
+; GCN-NEXT: v_mov_b32_e32 v22, 0
+; GCN-NEXT: ds_load_tr16_b128 v[32:35], v12
+; GCN-NEXT: ds_load_tr16_b128 v[36:39], v12 offset:64
+; GCN-NEXT: ds_load_tr16_b128 v[40:43], v12 offset:128
+; GCN-NEXT: ds_load_tr16_b128 v[44:47], v12 offset:192
+; GCN-NEXT: ds_load_tr16_b128 v[48:51], v12 offset:256
+; GCN-NEXT: ds_load_tr16_b128 v[52:55], v12 offset:320
+; GCN-NEXT: ds_load_tr16_b128 v[56:59], v12 offset:384
+; GCN-NEXT: ds_load_tr16_b128 v[60:63], v12 offset:448
+; GCN-NEXT: v_dual_mov_b32 v12, 0 :: v_dual_mov_b32 v23, 0
+; GCN-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v25, 0
+; GCN-NEXT: v_dual_mov_b32 v26, 0 :: v_dual_mov_b32 v27, 0
+; GCN-NEXT: v_dual_mov_b32 v28, 0 :: v_dual_mov_b32 v29, 0
+; GCN-NEXT: v_dual_mov_b32 v30, 0 :: v_dual_mov_b32 v31, 0
+; GCN-NEXT: s_cselect_b32 s3, -1, 0
; GCN-NEXT: s_mov_b32 s8, 0
+; GCN-NEXT: s_add_co_i32 s0, s2, s1
+; GCN-NEXT: s_xor_b32 s2, s3, -1
; GCN-NEXT: s_mov_b32 s9, s8
; GCN-NEXT: s_mov_b32 s10, s8
; GCN-NEXT: s_mov_b32 s11, s8
@@ -724,36 +751,6 @@ define amdgpu_kernel void @ds_wmma_block_carried(ptr addrspace(3) %base, ptr add
; GCN-NEXT: s_mov_b32 s13, s8
; GCN-NEXT: s_mov_b32 s14, s8
; GCN-NEXT: s_mov_b32 s15, s8
-; GCN-NEXT: s_wait_kmcnt 0x0
-; GCN-NEXT: s_bitcmp1_b32 s0, 0
-; GCN-NEXT: v_dual_mov_b32 v7, s2 :: v_dual_mov_b32 v1, v0
-; GCN-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, v0
-; GCN-NEXT: v_mov_b32_e32 v4, v0
-; GCN-NEXT: ds_load_tr16_b128 v[32:35], v7
-; GCN-NEXT: ds_load_tr16_b128 v[36:39], v7 offset:64
-; GCN-NEXT: ds_load_tr16_b128 v[40:43], v7 offset:128
-; GCN-NEXT: ds_load_tr16_b128 v[44:47], v7 offset:192
-; GCN-NEXT: ds_load_tr16_b128 v[48:51], v7 offset:256
-; GCN-NEXT: ds_load_tr16_b128 v[52:55], v7 offset:320
-; GCN-NEXT: ds_load_tr16_b128 v[56:59], v7 offset:384
-; GCN-NEXT: ds_load_tr16_b128 v[60:63], v7 offset:448
-; GCN-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0
-; GCN-NEXT: v_dual_mov_b32 v7, v0 :: v_dual_mov_b32 v8, v0
-; GCN-NEXT: v_dual_mov_b32 v9, v0 :: v_dual_mov_b32 v10, v0
-; GCN-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0
-; GCN-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0
-; GCN-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v16, v0
-; GCN-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v0
-; GCN-NEXT: v_dual_mov_b32 v19, v0 :: v_dual_mov_b32 v20, v0
-; GCN-NEXT: v_dual_mov_b32 v21, v0 :: v_dual_mov_b32 v22, v0
-; GCN-NEXT: v_dual_mov_b32 v23, v0 :: v_dual_mov_b32 v24, v0
-; GCN-NEXT: v_dual_mov_b32 v25, v0 :: v_dual_mov_b32 v26, v0
-; GCN-NEXT: v_dual_mov_b32 v27, v0 :: v_dual_mov_b32 v28, v0
-; GCN-NEXT: v_dual_mov_b32 v29, v0 :: v_dual_mov_b32 v30, v0
-; GCN-NEXT: v_mov_b32_e32 v31, v0
-; GCN-NEXT: s_cselect_b32 s3, -1, 0
-; GCN-NEXT: s_add_co_i32 s0, s2, s1
-; GCN-NEXT: s_xor_b32 s2, s3, -1
; GCN-NEXT: s_wait_dscnt 0x0
; GCN-NEXT: .LBB2_1: ; %loop
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
diff --git a/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll b/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
index 873d28b54337b..e5687b349c1b0 100644
--- a/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
+++ b/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
@@ -984,21 +984,21 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: s_movk_i32 s4, 0x207
; GCN-NEXT: v_cmp_gt_i32_e32 vcc, s4, v0
+; GCN-NEXT: s_mov_b32 s8, 0
; GCN-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v0
-; GCN-NEXT: s_mov_b64 s[8:9], 0
-; GCN-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NEXT: s_mov_b64 s[12:13], 0
; GCN-NEXT: s_mov_b64 s[6:7], 0
; GCN-NEXT: s_branch .LBB5_3
; GCN-NEXT: .LBB5_1: ; %Flow
; GCN-NEXT: ; in Loop: Header=BB5_3 Depth=1
-; GCN-NEXT: s_or_b64 exec, exec, s[12:13]
+; GCN-NEXT: s_or_b64 exec, exec, s[10:11]
; GCN-NEXT: .LBB5_2: ; %bb10
; GCN-NEXT: ; in Loop: Header=BB5_3 Depth=1
-; GCN-NEXT: s_or_b64 exec, exec, s[10:11]
+; GCN-NEXT: s_or_b64 exec, exec, s[14:15]
; GCN-NEXT: s_and_b64 s[6:7], exec, s[4:5]
-; GCN-NEXT: s_or_b64 s[8:9], s[6:7], s[8:9]
+; GCN-NEXT: s_or_b64 s[12:13], s[6:7], s[12:13]
; GCN-NEXT: s_mov_b64 s[6:7], 0
-; GCN-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GCN-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GCN-NEXT: s_cbranch_execz .LBB5_7
; GCN-NEXT: .LBB5_3: ; %bb1
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -1009,45 +1009,43 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-NEXT: ; %bb.4: ; %bb2
; GCN-NEXT: ; in Loop: Header=BB5_3 Depth=1
; GCN-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN-NEXT: v_mov_b32_e32 v2, v1
-; GCN-NEXT: v_mov_b32_e32 v3, v1
-; GCN-NEXT: v_mov_b32_e32 v4, v1
-; GCN-NEXT: v_mov_b32_e32 v5, v4
-; GCN-NEXT: v_mov_b32_e32 v4, v3
-; GCN-NEXT: v_mov_b32_e32 v3, v2
-; GCN-NEXT: v_mov_b32_e32 v2, v1
-; GCN-NEXT: s_and_saveexec_b64 s[10:11], s[4:5]
+; GCN-NEXT: s_mov_b32 s9, s8
+; GCN-NEXT: s_mov_b32 s10, s8
+; GCN-NEXT: s_mov_b32 s11, s8
+; GCN-NEXT: v_mov_b32_e32 v0, s8
+; GCN-NEXT: v_mov_b32_e32 v1, s9
+; GCN-NEXT: v_mov_b32_e32 v2, s10
+; GCN-NEXT: v_mov_b32_e32 v3, s11
+; GCN-NEXT: s_and_saveexec_b64 s[14:15], s[4:5]
; GCN-NEXT: s_cbranch_execz .LBB5_2
; GCN-NEXT: ; %bb.5: ; %bb4
; GCN-NEXT: ; in Loop: Header=BB5_3 Depth=1
; GCN-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
-; GCN-NEXT: v_mov_b32_e32 v2, v1
-; GCN-NEXT: v_mov_b32_e32 v3, v1
-; GCN-NEXT: v_mov_b32_e32 v4, v1
-; GCN-NEXT: v_mov_b32_e32 v5, v4
-; GCN-NEXT: v_mov_b32_e32 v4, v3
-; GCN-NEXT: v_mov_b32_e32 v3, v2
-; GCN-NEXT: v_mov_b32_e32 v2, v1
+; GCN-NEXT: v_mov_b32_e32 v1, s9
+; GCN-NEXT: v_mov_b32_e32 v2, s10
+; GCN-NEXT: v_mov_b32_e32 v3, s11
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_cmp_gt_f32_e64 s[6:7], 0, v0
-; GCN-NEXT: s_and_saveexec_b64 s[12:13], s[6:7]
+; GCN-NEXT: v_mov_b32_e32 v0, s8
+; GCN-NEXT: s_and_saveexec_b64 s[10:11], s[6:7]
; GCN-NEXT: s_cbranch_execz .LBB5_1
; GCN-NEXT: ; %bb.6: ; %bb8
; GCN-NEXT: ; in Loop: Header=BB5_3 Depth=1
-; GCN-NEXT: v_mov_b32_e32 v5, v3
-; GCN-NEXT: v_mov_b32_e32 v4, v2
-; GCN-NEXT: v_mov_b32_e32 v3, v1
-; GCN-NEXT: v_mov_b32_e32 v2, v0
+; GCN-NEXT: s_mov_b32 s9, s8
+; GCN-NEXT: v_mov_b32_e32 v0, s8
+; GCN-NEXT: v_mov_b32_e32 v1, s9
+; GCN-NEXT: v_mov_b32_e32 v2, s10
+; GCN-NEXT: v_mov_b32_e32 v3, s11
; GCN-NEXT: s_branch .LBB5_1
; GCN-NEXT: .LBB5_7: ; %bb12
-; GCN-NEXT: s_or_b64 exec, exec, s[8:9]
-; GCN-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen
-; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: s_or_b64 exec, exec, s[12:13]
; GCN-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_store_dword v0, v0, s[0:3], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GCN-NEXT: s_setpc_b64 s[30:31]
;
@@ -1055,59 +1053,59 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-O0: ; %bb.0: ; %bb
; GCN-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GCN-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
+; GCN-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:72 ; 4-byte Folded Spill
; GCN-O0-NEXT: s_mov_b64 exec, s[4:5]
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GCN-O0-NEXT: s_mov_b64 s[4:5], 0
; GCN-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
-; GCN-O0-NEXT: ; implicit-def: $vgpr7 : SGPR spill to VGPR lane
+; GCN-O0-NEXT: ; implicit-def: $vgpr6 : SGPR spill to VGPR lane
; GCN-O0-NEXT: s_waitcnt expcnt(1)
-; GCN-O0-NEXT: v_writelane_b32 v7, s6, 0
-; GCN-O0-NEXT: v_writelane_b32 v7, s7, 1
-; GCN-O0-NEXT: v_writelane_b32 v7, s4, 2
-; GCN-O0-NEXT: v_writelane_b32 v7, s5, 3
-; GCN-O0-NEXT: s_or_saveexec_b64 s[12:13], -1
-; GCN-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 ; 4-byte Folded Spill
-; GCN-O0-NEXT: s_mov_b64 exec, s[12:13]
+; GCN-O0-NEXT: v_writelane_b32 v6, s6, 0
+; GCN-O0-NEXT: v_writelane_b32 v6, s7, 1
+; GCN-O0-NEXT: v_writelane_b32 v6, s4, 2
+; GCN-O0-NEXT: v_writelane_b32 v6, s5, 3
+; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
+; GCN-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-O0-NEXT: s_mov_b64 exec, s[14:15]
; GCN-O0-NEXT: .LBB5_1: ; %bb1
; GCN-O0-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN-O0-NEXT: s_or_saveexec_b64 s[12:13], -1
+; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
; GCN-O0-NEXT: s_waitcnt expcnt(0)
-; GCN-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_mov_b64 exec, s[12:13]
+; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-O0-NEXT: s_mov_b64 exec, s[14:15]
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
-; GCN-O0-NEXT: v_readlane_b32 s8, v7, 2
-; GCN-O0-NEXT: v_readlane_b32 s9, v7, 3
-; GCN-O0-NEXT: v_readlane_b32 s6, v7, 0
-; GCN-O0-NEXT: v_readlane_b32 s7, v7, 1
-; GCN-O0-NEXT: v_writelane_b32 v7, s6, 4
-; GCN-O0-NEXT: v_writelane_b32 v7, s7, 5
+; GCN-O0-NEXT: v_readlane_b32 s8, v6, 2
+; GCN-O0-NEXT: v_readlane_b32 s9, v6, 3
+; GCN-O0-NEXT: v_readlane_b32 s6, v6, 0
+; GCN-O0-NEXT: v_readlane_b32 s7, v6, 1
+; GCN-O0-NEXT: v_writelane_b32 v6, s6, 4
+; GCN-O0-NEXT: v_writelane_b32 v6, s7, 5
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_mov_b32 s4, 0x207
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_cmp_lt_i32_e64 s[4:5], v0, s4
; GCN-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
-; GCN-O0-NEXT: v_writelane_b32 v7, s4, 6
-; GCN-O0-NEXT: v_writelane_b32 v7, s5, 7
-; GCN-O0-NEXT: v_writelane_b32 v7, s6, 0
-; GCN-O0-NEXT: v_writelane_b32 v7, s7, 1
+; GCN-O0-NEXT: v_writelane_b32 v6, s4, 6
+; GCN-O0-NEXT: v_writelane_b32 v6, s5, 7
+; GCN-O0-NEXT: v_writelane_b32 v6, s6, 0
+; GCN-O0-NEXT: v_writelane_b32 v6, s7, 1
; GCN-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
-; GCN-O0-NEXT: v_writelane_b32 v7, s6, 2
-; GCN-O0-NEXT: v_writelane_b32 v7, s7, 3
-; GCN-O0-NEXT: s_or_saveexec_b64 s[12:13], -1
-; GCN-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 ; 4-byte Folded Spill
-; GCN-O0-NEXT: s_mov_b64 exec, s[12:13]
+; GCN-O0-NEXT: v_writelane_b32 v6, s6, 2
+; GCN-O0-NEXT: v_writelane_b32 v6, s7, 3
+; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
+; GCN-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-O0-NEXT: s_mov_b64 exec, s[14:15]
; GCN-O0-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN-O0-NEXT: s_cbranch_execnz .LBB5_1
; GCN-O0-NEXT: ; %bb.2: ; %bb2
; GCN-O0-NEXT: ; in Loop: Header=BB5_1 Depth=1
-; GCN-O0-NEXT: s_or_saveexec_b64 s[12:13], -1
+; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
; GCN-O0-NEXT: s_waitcnt expcnt(0)
-; GCN-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_mov_b64 exec, s[12:13]
+; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-O0-NEXT: s_mov_b64 exec, s[14:15]
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
-; GCN-O0-NEXT: v_readlane_b32 s4, v7, 6
-; GCN-O0-NEXT: v_readlane_b32 s5, v7, 7
+; GCN-O0-NEXT: v_readlane_b32 s4, v6, 6
+; GCN-O0-NEXT: v_readlane_b32 s5, v6, 7
; GCN-O0-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_mov_b32 s4, 0
@@ -1115,60 +1113,60 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-O0-NEXT: v_cmp_eq_u32_e64 s[6:7], v0, s4
; GCN-O0-NEXT: s_mov_b64 s[4:5], -1
; GCN-O0-NEXT: s_xor_b64 s[4:5], s[6:7], s[4:5]
-; GCN-O0-NEXT: v_writelane_b32 v7, s4, 8
-; GCN-O0-NEXT: v_writelane_b32 v7, s5, 9
+; GCN-O0-NEXT: v_writelane_b32 v6, s4, 8
+; GCN-O0-NEXT: v_writelane_b32 v6, s5, 9
; GCN-O0-NEXT: s_mov_b32 s4, 0
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v6, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v5, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v4, s4
-; GCN-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v5
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v4
+; GCN-O0-NEXT: s_mov_b32 s8, s4
+; GCN-O0-NEXT: s_mov_b32 s9, s4
+; GCN-O0-NEXT: s_mov_b32 s10, s4
+; GCN-O0-NEXT: s_mov_b32 s11, s4
+; GCN-O0-NEXT: v_mov_b32_e32 v0, s8
+; GCN-O0-NEXT: v_mov_b32_e32 v1, s9
+; GCN-O0-NEXT: v_mov_b32_e32 v2, s10
+; GCN-O0-NEXT: v_mov_b32_e32 v3, s11
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:12 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:16 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; GCN-O0-NEXT: s_mov_b64 s[4:5], exec
-; GCN-O0-NEXT: v_writelane_b32 v7, s4, 10
-; GCN-O0-NEXT: v_writelane_b32 v7, s5, 11
-; GCN-O0-NEXT: s_or_saveexec_b64 s[12:13], -1
-; GCN-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 ; 4-byte Folded Spill
-; GCN-O0-NEXT: s_mov_b64 exec, s[12:13]
+; GCN-O0-NEXT: v_writelane_b32 v6, s4, 10
+; GCN-O0-NEXT: v_writelane_b32 v6, s5, 11
+; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
+; GCN-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-O0-NEXT: s_mov_b64 exec, s[14:15]
; GCN-O0-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
; GCN-O0-NEXT: s_mov_b64 exec, s[4:5]
; GCN-O0-NEXT: s_cbranch_execz .LBB5_5
; GCN-O0-NEXT: ; %bb.3: ; %bb4
; GCN-O0-NEXT: ; in Loop: Header=BB5_1 Depth=1
-; GCN-O0-NEXT: s_or_saveexec_b64 s[12:13], -1
+; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
; GCN-O0-NEXT: s_waitcnt expcnt(0)
-; GCN-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_mov_b64 exec, s[12:13]
+; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-O0-NEXT: s_mov_b64 exec, s[14:15]
; GCN-O0-NEXT: ; implicit-def: $sgpr4
; GCN-O0-NEXT: v_mov_b32_e32 v0, s4
; GCN-O0-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
; GCN-O0-NEXT: s_mov_b32 s4, 0
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: v_cmp_lt_f32_e64 s[6:7], v0, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v0, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v6, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v5, s4
-; GCN-O0-NEXT: v_mov_b32_e32 v4, s4
-; GCN-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1_vgpr2_vgpr3 killed $exec
-; GCN-O0-NEXT: v_mov_b32_e32 v1, v6
-; GCN-O0-NEXT: v_mov_b32_e32 v2, v5
-; GCN-O0-NEXT: v_mov_b32_e32 v3, v4
+; GCN-O0-NEXT: s_mov_b32 s8, s4
+; GCN-O0-NEXT: s_mov_b32 s9, s4
+; GCN-O0-NEXT: s_mov_b32 s10, s4
+; GCN-O0-NEXT: s_mov_b32 s11, s4
+; GCN-O0-NEXT: v_mov_b32_e32 v0, s8
+; GCN-O0-NEXT: v_mov_b32_e32 v1, s9
+; GCN-O0-NEXT: v_mov_b32_e32 v2, s10
+; GCN-O0-NEXT: v_mov_b32_e32 v3, s11
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
; GCN-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:36 ; 4-byte Folded Spill
; GCN-O0-NEXT: s_mov_b64 s[4:5], exec
-; GCN-O0-NEXT: v_writelane_b32 v7, s4, 12
-; GCN-O0-NEXT: v_writelane_b32 v7, s5, 13
-; GCN-O0-NEXT: s_or_saveexec_b64 s[12:13], -1
-; GCN-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 ; 4-byte Folded Spill
-; GCN-O0-NEXT: s_mov_b64 exec, s[12:13]
+; GCN-O0-NEXT: v_writelane_b32 v6, s4, 12
+; GCN-O0-NEXT: v_writelane_b32 v6, s5, 13
+; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
+; GCN-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-O0-NEXT: s_mov_b64 exec, s[14:15]
; GCN-O0-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
; GCN-O0-NEXT: s_mov_b64 exec, s[4:5]
; GCN-O0-NEXT: s_cbranch_execz .LBB5_6
@@ -1192,13 +1190,13 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-O0-NEXT: s_branch .LBB5_6
; GCN-O0-NEXT: .LBB5_5: ; %Flow2
; GCN-O0-NEXT: ; in Loop: Header=BB5_1 Depth=1
-; GCN-O0-NEXT: s_or_saveexec_b64 s[12:13], -1
+; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
; GCN-O0-NEXT: s_waitcnt expcnt(0)
-; GCN-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_mov_b64 exec, s[12:13]
+; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-O0-NEXT: s_mov_b64 exec, s[14:15]
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
-; GCN-O0-NEXT: v_readlane_b32 s4, v7, 10
-; GCN-O0-NEXT: v_readlane_b32 s5, v7, 11
+; GCN-O0-NEXT: v_readlane_b32 s4, v6, 10
+; GCN-O0-NEXT: v_readlane_b32 s5, v6, 11
; GCN-O0-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
@@ -1215,13 +1213,13 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-O0-NEXT: s_branch .LBB5_7
; GCN-O0-NEXT: .LBB5_6: ; %Flow
; GCN-O0-NEXT: ; in Loop: Header=BB5_1 Depth=1
-; GCN-O0-NEXT: s_or_saveexec_b64 s[12:13], -1
+; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
; GCN-O0-NEXT: s_waitcnt expcnt(0)
-; GCN-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_mov_b64 exec, s[12:13]
+; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-O0-NEXT: s_mov_b64 exec, s[14:15]
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
-; GCN-O0-NEXT: v_readlane_b32 s4, v7, 12
-; GCN-O0-NEXT: v_readlane_b32 s5, v7, 13
+; GCN-O0-NEXT: v_readlane_b32 s4, v6, 12
+; GCN-O0-NEXT: v_readlane_b32 s5, v6, 13
; GCN-O0-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:24 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:28 ; 4-byte Folded Reload
@@ -1238,52 +1236,52 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-O0-NEXT: s_branch .LBB5_5
; GCN-O0-NEXT: .LBB5_7: ; %bb10
; GCN-O0-NEXT: ; in Loop: Header=BB5_1 Depth=1
-; GCN-O0-NEXT: s_or_saveexec_b64 s[12:13], -1
-; GCN-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_mov_b64 exec, s[12:13]
+; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
+; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-O0-NEXT: s_mov_b64 exec, s[14:15]
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
-; GCN-O0-NEXT: v_readlane_b32 s6, v7, 8
-; GCN-O0-NEXT: v_readlane_b32 s7, v7, 9
+; GCN-O0-NEXT: v_readlane_b32 s6, v6, 8
+; GCN-O0-NEXT: v_readlane_b32 s7, v6, 9
; GCN-O0-NEXT: s_mov_b64 s[4:5], -1
-; GCN-O0-NEXT: v_writelane_b32 v7, s4, 14
-; GCN-O0-NEXT: v_writelane_b32 v7, s5, 15
+; GCN-O0-NEXT: v_writelane_b32 v6, s4, 14
+; GCN-O0-NEXT: v_writelane_b32 v6, s5, 15
; GCN-O0-NEXT: s_mov_b64 s[4:5], exec
-; GCN-O0-NEXT: v_writelane_b32 v7, s4, 16
-; GCN-O0-NEXT: v_writelane_b32 v7, s5, 17
-; GCN-O0-NEXT: s_or_saveexec_b64 s[12:13], -1
-; GCN-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 ; 4-byte Folded Spill
-; GCN-O0-NEXT: s_mov_b64 exec, s[12:13]
+; GCN-O0-NEXT: v_writelane_b32 v6, s4, 16
+; GCN-O0-NEXT: v_writelane_b32 v6, s5, 17
+; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
+; GCN-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-O0-NEXT: s_mov_b64 exec, s[14:15]
; GCN-O0-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
; GCN-O0-NEXT: s_mov_b64 exec, s[4:5]
; GCN-O0-NEXT: s_cbranch_execz .LBB5_9
; GCN-O0-NEXT: ; %bb.8: ; %Flow1
; GCN-O0-NEXT: ; in Loop: Header=BB5_1 Depth=1
-; GCN-O0-NEXT: s_or_saveexec_b64 s[12:13], -1
+; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
; GCN-O0-NEXT: s_waitcnt expcnt(0)
-; GCN-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_mov_b64 exec, s[12:13]
+; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-O0-NEXT: s_mov_b64 exec, s[14:15]
; GCN-O0-NEXT: s_mov_b64 s[4:5], 0
; GCN-O0-NEXT: s_xor_b64 s[4:5], exec, -1
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
-; GCN-O0-NEXT: v_writelane_b32 v7, s4, 14
-; GCN-O0-NEXT: v_writelane_b32 v7, s5, 15
-; GCN-O0-NEXT: s_or_saveexec_b64 s[12:13], -1
-; GCN-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 ; 4-byte Folded Spill
-; GCN-O0-NEXT: s_mov_b64 exec, s[12:13]
+; GCN-O0-NEXT: v_writelane_b32 v6, s4, 14
+; GCN-O0-NEXT: v_writelane_b32 v6, s5, 15
+; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
+; GCN-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-O0-NEXT: s_mov_b64 exec, s[14:15]
; GCN-O0-NEXT: .LBB5_9: ; %Flow3
; GCN-O0-NEXT: ; in Loop: Header=BB5_1 Depth=1
-; GCN-O0-NEXT: s_or_saveexec_b64 s[12:13], -1
+; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
; GCN-O0-NEXT: s_waitcnt expcnt(0)
-; GCN-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_mov_b64 exec, s[12:13]
+; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-O0-NEXT: s_mov_b64 exec, s[14:15]
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
-; GCN-O0-NEXT: v_readlane_b32 s8, v7, 16
-; GCN-O0-NEXT: v_readlane_b32 s9, v7, 17
+; GCN-O0-NEXT: v_readlane_b32 s8, v6, 16
+; GCN-O0-NEXT: v_readlane_b32 s9, v6, 17
; GCN-O0-NEXT: s_or_b64 exec, exec, s[8:9]
-; GCN-O0-NEXT: v_readlane_b32 s6, v7, 4
-; GCN-O0-NEXT: v_readlane_b32 s7, v7, 5
-; GCN-O0-NEXT: v_readlane_b32 s4, v7, 14
-; GCN-O0-NEXT: v_readlane_b32 s5, v7, 15
+; GCN-O0-NEXT: v_readlane_b32 s6, v6, 4
+; GCN-O0-NEXT: v_readlane_b32 s7, v6, 5
+; GCN-O0-NEXT: v_readlane_b32 s4, v6, 14
+; GCN-O0-NEXT: v_readlane_b32 s5, v6, 15
; GCN-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
; GCN-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
@@ -1292,16 +1290,16 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-O0-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
; GCN-O0-NEXT: s_mov_b64 s[6:7], 0
; GCN-O0-NEXT: s_mov_b64 s[8:9], s[4:5]
-; GCN-O0-NEXT: v_writelane_b32 v7, s8, 0
-; GCN-O0-NEXT: v_writelane_b32 v7, s9, 1
-; GCN-O0-NEXT: v_writelane_b32 v7, s6, 2
-; GCN-O0-NEXT: v_writelane_b32 v7, s7, 3
+; GCN-O0-NEXT: v_writelane_b32 v6, s8, 0
+; GCN-O0-NEXT: v_writelane_b32 v6, s9, 1
+; GCN-O0-NEXT: v_writelane_b32 v6, s6, 2
+; GCN-O0-NEXT: v_writelane_b32 v6, s7, 3
; GCN-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
-; GCN-O0-NEXT: v_writelane_b32 v7, s6, 18
-; GCN-O0-NEXT: v_writelane_b32 v7, s7, 19
-; GCN-O0-NEXT: s_or_saveexec_b64 s[12:13], -1
-; GCN-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 ; 4-byte Folded Spill
-; GCN-O0-NEXT: s_mov_b64 exec, s[12:13]
+; GCN-O0-NEXT: v_writelane_b32 v6, s6, 18
+; GCN-O0-NEXT: v_writelane_b32 v6, s7, 19
+; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
+; GCN-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
+; GCN-O0-NEXT: s_mov_b64 exec, s[14:15]
; GCN-O0-NEXT: s_waitcnt vmcnt(4)
; GCN-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:56 ; 4-byte Folded Spill
; GCN-O0-NEXT: s_waitcnt vmcnt(4)
@@ -1313,13 +1311,13 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-O0-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN-O0-NEXT: s_cbranch_execnz .LBB5_1
; GCN-O0-NEXT: ; %bb.10: ; %bb12
-; GCN-O0-NEXT: s_or_saveexec_b64 s[12:13], -1
+; GCN-O0-NEXT: s_or_saveexec_b64 s[14:15], -1
; GCN-O0-NEXT: s_waitcnt expcnt(4)
-; GCN-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload
-; GCN-O0-NEXT: s_mov_b64 exec, s[12:13]
+; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
+; GCN-O0-NEXT: s_mov_b64 exec, s[14:15]
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
-; GCN-O0-NEXT: v_readlane_b32 s4, v7, 18
-; GCN-O0-NEXT: v_readlane_b32 s5, v7, 19
+; GCN-O0-NEXT: v_readlane_b32 s4, v6, 18
+; GCN-O0-NEXT: v_readlane_b32 s5, v6, 19
; GCN-O0-NEXT: s_or_b64 exec, exec, s[4:5]
; GCN-O0-NEXT: ; %bb.11: ; %bb12
; GCN-O0-NEXT: s_waitcnt expcnt(3)
@@ -1352,7 +1350,7 @@ define void @scc_liveness(i32 %arg) local_unnamed_addr #0 {
; GCN-O0-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GCN-O0-NEXT: s_waitcnt vmcnt(0)
; GCN-O0-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; GCN-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
+; GCN-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:72 ; 4-byte Folded Reload
; GCN-O0-NEXT: s_mov_b64 exec, s[4:5]
; GCN-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GCN-O0-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll b/llvm/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll
index dcc3fa5bf13c8..d3694a29ece9d 100644
--- a/llvm/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/control-flow-fastregalloc.ll
@@ -17,7 +17,7 @@
; Spill load
; GCN: buffer_store_dword [[LOAD0]], off, s[0:3], 0 offset:[[LOAD0_OFFSET:[0-9]+]] ; 4-byte Folded Spill
-; GCN: v_cmp_eq_u32_e64 [[CMP0:s\[[0-9]+:[0-9]\]]], v{{[0-9]+}}, v{{[0-9]+}}
+; GCN: v_cmp_eq_u32_e64 [[CMP0:s\[[0-9]+:[0-9]\]]], v{{[0-9]+}}, s{{[0-9]+}}
; Spill saved exec
; GCN: s_mov_b64 s[[[SAVEEXEC_LO:[0-9]+]]:[[SAVEEXEC_HI:[0-9]+]]], exec
diff --git a/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll b/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll
index 4ffff7bd28ad4..5d2e84db890fe 100644
--- a/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll
@@ -203,8 +203,8 @@ define amdgpu_kernel void @v3i16_registers(i1 %cond) #0 {
; GCN-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GCN-NEXT: s_branch .LBB4_3
; GCN-NEXT: .LBB4_2:
-; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: .LBB4_3: ; %if.end
; GCN-NEXT: global_store_short v[0:1], v1, off
; GCN-NEXT: global_store_dword v[0:1], v0, off
@@ -255,8 +255,8 @@ define amdgpu_kernel void @v3f16_registers(i1 %cond) #0 {
; GCN-NEXT: s_swappc_b64 s[30:31], s[18:19]
; GCN-NEXT: s_branch .LBB5_3
; GCN-NEXT: .LBB5_2:
-; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: .LBB5_3: ; %if.end
; GCN-NEXT: global_store_short v[0:1], v1, off
; GCN-NEXT: global_store_dword v[0:1], v0, off
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-crash.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-crash.ll
index 3888365116799..7fdb5eb3f8188 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-crash.ll
@@ -11,7 +11,7 @@ define void @main(float %arg) {
; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
; CHECK-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
- ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
; CHECK-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 $exec_lo, [[S_MOV_B32_1]], implicit-def dead $scc
; CHECK-NEXT: $vcc_lo = COPY [[S_AND_B32_]]
; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
diff --git a/llvm/test/CodeGen/AMDGPU/div_i128.ll b/llvm/test/CodeGen/AMDGPU/div_i128.ll
index 44db4fd1fa82f..ce6a6f596efd3 100644
--- a/llvm/test/CodeGen/AMDGPU/div_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_i128.ll
@@ -863,12 +863,12 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:308 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:312 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
@@ -878,7 +878,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-O0-NEXT: s_mov_b32 s4, 64
; GFX9-O0-NEXT: v_sub_u32_e64 v20, s4, v4
-; GFX9-O0-NEXT: v_lshlrev_b64 v[20:21], v20, v[16:17]
+; GFX9-O0-NEXT: v_lshlrev_b64 v[20:21], v20, v[14:15]
; GFX9-O0-NEXT: v_mov_b32_e32 v22, v21
; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v22
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
@@ -889,7 +889,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v22, v7
; GFX9-O0-NEXT: v_cmp_lt_u32_e64 s[6:7], v4, s4
; GFX9-O0-NEXT: v_sub_u32_e64 v5, v4, s4
-; GFX9-O0-NEXT: v_lshrrev_b64 v[20:21], v5, v[16:17]
+; GFX9-O0-NEXT: v_lshrrev_b64 v[20:21], v5, v[14:15]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v21
; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v22, s[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6
@@ -907,46 +907,47 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5]
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5
-; GFX9-O0-NEXT: v_lshrrev_b64 v[4:5], v4, v[16:17]
-; GFX9-O0-NEXT: v_mov_b32_e32 v17, v5
+; GFX9-O0-NEXT: v_lshrrev_b64 v[4:5], v4, v[14:15]
+; GFX9-O0-NEXT: v_mov_b32_e32 v15, v5
; GFX9-O0-NEXT: s_mov_b64 s[4:5], 0
; GFX9-O0-NEXT: s_mov_b32 s8, s5
-; GFX9-O0-NEXT: v_mov_b32_e32 v16, s8
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v16, v16, v17, s[6:7]
+; GFX9-O0-NEXT: v_mov_b32_e32 v14, s8
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v14, v14, v15, s[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v4
; GFX9-O0-NEXT: s_mov_b32 s8, s4
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s8
; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[6:7]
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v5, v16
+; GFX9-O0-NEXT: v_mov_b32_e32 v5, v14
; GFX9-O0-NEXT: s_mov_b64 s[10:11], -1
; GFX9-O0-NEXT: s_mov_b32 s9, s10
-; GFX9-O0-NEXT: v_mov_b32_e32 v16, v14
-; GFX9-O0-NEXT: v_add_co_u32_e64 v18, s[6:7], v16, s9
-; GFX9-O0-NEXT: s_mov_b32 s8, s11
-; GFX9-O0-NEXT: v_mov_b32_e32 v14, v15
-; GFX9-O0-NEXT: v_mov_b32_e32 v15, s8
-; GFX9-O0-NEXT: v_addc_co_u32_e64 v14, s[6:7], v14, v15, s[6:7]
-; GFX9-O0-NEXT: ; kill: def $vgpr18 killed $vgpr18 def $vgpr18_vgpr19 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v19, v14
; GFX9-O0-NEXT: v_mov_b32_e32 v14, v12
-; GFX9-O0-NEXT: v_mov_b32_e32 v15, s9
-; GFX9-O0-NEXT: v_addc_co_u32_e64 v16, s[6:7], v14, v15, s[6:7]
+; GFX9-O0-NEXT: v_add_co_u32_e64 v14, s[6:7], v14, s9
+; GFX9-O0-NEXT: s_mov_b32 s8, s11
; GFX9-O0-NEXT: v_mov_b32_e32 v12, v13
; GFX9-O0-NEXT: v_mov_b32_e32 v13, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v12, s[6:7], v12, v13, s[6:7]
-; GFX9-O0-NEXT: ; kill: def $vgpr16 killed $vgpr16 def $vgpr16_vgpr17 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v17, v12
-; GFX9-O0-NEXT: v_mov_b32_e32 v13, s5
-; GFX9-O0-NEXT: v_mov_b32_e32 v12, s4
-; GFX9-O0-NEXT: v_mov_b32_e32 v15, s5
-; GFX9-O0-NEXT: v_mov_b32_e32 v14, s4
-; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec
+; GFX9-O0-NEXT: v_mov_b32_e32 v15, v12
+; GFX9-O0-NEXT: v_mov_b32_e32 v12, v16
+; GFX9-O0-NEXT: v_mov_b32_e32 v13, s9
+; GFX9-O0-NEXT: v_addc_co_u32_e64 v12, s[6:7], v12, v13, s[6:7]
+; GFX9-O0-NEXT: v_mov_b32_e32 v13, v17
+; GFX9-O0-NEXT: v_mov_b32_e32 v16, s8
+; GFX9-O0-NEXT: v_addc_co_u32_e64 v16, s[6:7], v13, v16, s[6:7]
+; GFX9-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
+; GFX9-O0-NEXT: v_mov_b32_e32 v13, v16
+; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
-; GFX9-O0-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
-; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: v_mov_b32_e32 v15, s7
+; GFX9-O0-NEXT: v_mov_b32_e32 v14, s6
+; GFX9-O0-NEXT: v_mov_b32_e32 v12, 0
+; GFX9-O0-NEXT: v_mov_b32_e32 v13, 0
; GFX9-O0-NEXT: v_writelane_b32 v31, s4, 8
; GFX9-O0-NEXT: v_writelane_b32 v31, s5, 9
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
@@ -1087,10 +1088,10 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v2
; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[4:5], v[0:1], s[6:7]
-; GFX9-O0-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-O0-NEXT: v_mov_b32_e32 v1, s7
; GFX9-O0-NEXT: v_mov_b32_e32 v2, s6
; GFX9-O0-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-O0-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, 0
; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:120 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:124 ; 4-byte Folded Spill
@@ -2320,11 +2321,11 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_cndmask_b32_e64 v1, v13, v1, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v0, v12, v0, s[4:5]
; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, -1, v7, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: v_mov_b32_e32 v16, 0
; GFX9-NEXT: v_mov_b32_e32 v17, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: v_mov_b32_e32 v13, 0
; GFX9-NEXT: .LBB1_3: ; %udiv-do-while
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -2965,12 +2966,12 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:284 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:288 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:292 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
@@ -2980,7 +2981,7 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-O0-NEXT: s_mov_b32 s4, 64
; GFX9-O0-NEXT: v_sub_u32_e64 v20, s4, v4
-; GFX9-O0-NEXT: v_lshlrev_b64 v[20:21], v20, v[16:17]
+; GFX9-O0-NEXT: v_lshlrev_b64 v[20:21], v20, v[14:15]
; GFX9-O0-NEXT: v_mov_b32_e32 v22, v21
; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v22
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
@@ -2991,7 +2992,7 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v22, v7
; GFX9-O0-NEXT: v_cmp_lt_u32_e64 s[6:7], v4, s4
; GFX9-O0-NEXT: v_sub_u32_e64 v5, v4, s4
-; GFX9-O0-NEXT: v_lshrrev_b64 v[20:21], v5, v[16:17]
+; GFX9-O0-NEXT: v_lshrrev_b64 v[20:21], v5, v[14:15]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v21
; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v22, s[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6
@@ -3009,46 +3010,47 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5]
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5
-; GFX9-O0-NEXT: v_lshrrev_b64 v[4:5], v4, v[16:17]
-; GFX9-O0-NEXT: v_mov_b32_e32 v17, v5
+; GFX9-O0-NEXT: v_lshrrev_b64 v[4:5], v4, v[14:15]
+; GFX9-O0-NEXT: v_mov_b32_e32 v15, v5
; GFX9-O0-NEXT: s_mov_b64 s[4:5], 0
; GFX9-O0-NEXT: s_mov_b32 s8, s5
-; GFX9-O0-NEXT: v_mov_b32_e32 v16, s8
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v16, v16, v17, s[6:7]
+; GFX9-O0-NEXT: v_mov_b32_e32 v14, s8
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v14, v14, v15, s[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v4
; GFX9-O0-NEXT: s_mov_b32 s8, s4
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s8
; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[6:7]
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v5, v16
+; GFX9-O0-NEXT: v_mov_b32_e32 v5, v14
; GFX9-O0-NEXT: s_mov_b64 s[10:11], -1
; GFX9-O0-NEXT: s_mov_b32 s9, s10
-; GFX9-O0-NEXT: v_mov_b32_e32 v16, v14
-; GFX9-O0-NEXT: v_add_co_u32_e64 v18, s[6:7], v16, s9
-; GFX9-O0-NEXT: s_mov_b32 s8, s11
-; GFX9-O0-NEXT: v_mov_b32_e32 v14, v15
-; GFX9-O0-NEXT: v_mov_b32_e32 v15, s8
-; GFX9-O0-NEXT: v_addc_co_u32_e64 v14, s[6:7], v14, v15, s[6:7]
-; GFX9-O0-NEXT: ; kill: def $vgpr18 killed $vgpr18 def $vgpr18_vgpr19 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v19, v14
; GFX9-O0-NEXT: v_mov_b32_e32 v14, v12
-; GFX9-O0-NEXT: v_mov_b32_e32 v15, s9
-; GFX9-O0-NEXT: v_addc_co_u32_e64 v16, s[6:7], v14, v15, s[6:7]
+; GFX9-O0-NEXT: v_add_co_u32_e64 v14, s[6:7], v14, s9
+; GFX9-O0-NEXT: s_mov_b32 s8, s11
; GFX9-O0-NEXT: v_mov_b32_e32 v12, v13
; GFX9-O0-NEXT: v_mov_b32_e32 v13, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v12, s[6:7], v12, v13, s[6:7]
-; GFX9-O0-NEXT: ; kill: def $vgpr16 killed $vgpr16 def $vgpr16_vgpr17 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v17, v12
-; GFX9-O0-NEXT: v_mov_b32_e32 v13, s5
-; GFX9-O0-NEXT: v_mov_b32_e32 v12, s4
-; GFX9-O0-NEXT: v_mov_b32_e32 v15, s5
-; GFX9-O0-NEXT: v_mov_b32_e32 v14, s4
-; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec
+; GFX9-O0-NEXT: v_mov_b32_e32 v15, v12
+; GFX9-O0-NEXT: v_mov_b32_e32 v12, v16
+; GFX9-O0-NEXT: v_mov_b32_e32 v13, s9
+; GFX9-O0-NEXT: v_addc_co_u32_e64 v12, s[6:7], v12, v13, s[6:7]
+; GFX9-O0-NEXT: v_mov_b32_e32 v13, v17
+; GFX9-O0-NEXT: v_mov_b32_e32 v16, s8
+; GFX9-O0-NEXT: v_addc_co_u32_e64 v16, s[6:7], v13, v16, s[6:7]
+; GFX9-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
+; GFX9-O0-NEXT: v_mov_b32_e32 v13, v16
+; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:256 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
-; GFX9-O0-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:248 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
-; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:252 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: v_mov_b32_e32 v15, s7
+; GFX9-O0-NEXT: v_mov_b32_e32 v14, s6
+; GFX9-O0-NEXT: v_mov_b32_e32 v12, 0
+; GFX9-O0-NEXT: v_mov_b32_e32 v13, 0
; GFX9-O0-NEXT: v_writelane_b32 v31, s4, 8
; GFX9-O0-NEXT: v_writelane_b32 v31, s5, 9
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
@@ -3189,10 +3191,10 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v2
; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[4:5], v[0:1], s[6:7]
-; GFX9-O0-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-O0-NEXT: v_mov_b32_e32 v1, s7
; GFX9-O0-NEXT: v_mov_b32_e32 v2, s6
; GFX9-O0-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-O0-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, 0
; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:96 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:100 ; 4-byte Folded Spill
diff --git a/llvm/test/CodeGen/AMDGPU/div_v2i128.ll b/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
index 19d9f94ee3d1b..14b58cc3ab050 100644
--- a/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
@@ -133,11 +133,11 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_addc_u32_e32 v33, vcc, -1, v28, vcc
; SDAG-NEXT: v_addc_u32_e32 v34, vcc, -1, v2, vcc
; SDAG-NEXT: v_addc_u32_e32 v35, vcc, -1, v3, vcc
-; SDAG-NEXT: v_mov_b32_e32 v16, 0
-; SDAG-NEXT: v_mov_b32_e32 v17, 0
; SDAG-NEXT: s_mov_b64 s[4:5], 0
; SDAG-NEXT: v_mov_b32_e32 v23, 0
; SDAG-NEXT: v_mov_b32_e32 v24, 0
+; SDAG-NEXT: v_mov_b32_e32 v16, 0
+; SDAG-NEXT: v_mov_b32_e32 v17, 0
; SDAG-NEXT: v_mov_b32_e32 v11, 0
; SDAG-NEXT: .LBB0_3: ; %udiv-do-while3
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -313,59 +313,59 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_cndmask_b32_e32 v13, v13, v15, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v15, v13, v5, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v5, v12, v14, vcc
-; SDAG-NEXT: v_cndmask_b32_e32 v9, 0, v9, vcc
-; SDAG-NEXT: v_cndmask_b32_e32 v8, 0, v8, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v17, 0, v9, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v16, 0, v8, vcc
; SDAG-NEXT: v_add_i32_e32 v30, vcc, -1, v22
; SDAG-NEXT: v_addc_u32_e32 v31, vcc, -1, v23, vcc
; SDAG-NEXT: v_addc_u32_e32 v32, vcc, -1, v2, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v14, v5, v4, s[4:5]
; SDAG-NEXT: v_addc_u32_e32 v33, vcc, -1, v3, vcc
+; SDAG-NEXT: s_mov_b64 s[10:11], 0
+; SDAG-NEXT: v_mov_b32_e32 v8, 0
+; SDAG-NEXT: v_mov_b32_e32 v9, 0
; SDAG-NEXT: v_mov_b32_e32 v4, 0
; SDAG-NEXT: v_mov_b32_e32 v5, 0
-; SDAG-NEXT: s_mov_b64 s[10:11], 0
-; SDAG-NEXT: v_mov_b32_e32 v16, 0
-; SDAG-NEXT: v_mov_b32_e32 v17, 0
; SDAG-NEXT: v_mov_b32_e32 v13, 0
; SDAG-NEXT: .LBB0_9: ; %udiv-do-while
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; SDAG-NEXT: v_lshl_b64 v[8:9], v[8:9], 1
+; SDAG-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
; SDAG-NEXT: v_lshrrev_b32_e32 v12, 31, v15
; SDAG-NEXT: v_lshl_b64 v[14:15], v[14:15], 1
-; SDAG-NEXT: v_or_b32_e32 v8, v8, v12
+; SDAG-NEXT: v_or_b32_e32 v16, v16, v12
; SDAG-NEXT: v_lshrrev_b32_e32 v12, 31, v7
-; SDAG-NEXT: v_lshl_b64 v[6:7], v[6:7], 1
; SDAG-NEXT: v_or_b32_e32 v14, v14, v12
; SDAG-NEXT: v_lshrrev_b32_e32 v12, 31, v11
-; SDAG-NEXT: v_or_b32_e32 v6, v6, v12
-; SDAG-NEXT: v_sub_i32_e32 v12, vcc, v30, v14
-; SDAG-NEXT: v_subb_u32_e32 v12, vcc, v31, v15, vcc
; SDAG-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
-; SDAG-NEXT: v_subb_u32_e32 v12, vcc, v32, v8, vcc
-; SDAG-NEXT: v_subb_u32_e32 v12, vcc, v33, v9, vcc
+; SDAG-NEXT: v_lshl_b64 v[6:7], v[6:7], 1
+; SDAG-NEXT: v_or_b32_e32 v10, v8, v10
+; SDAG-NEXT: v_sub_i32_e32 v8, vcc, v30, v14
+; SDAG-NEXT: v_subb_u32_e32 v8, vcc, v31, v15, vcc
+; SDAG-NEXT: v_subb_u32_e32 v8, vcc, v32, v16, vcc
+; SDAG-NEXT: v_subb_u32_e32 v8, vcc, v33, v17, vcc
; SDAG-NEXT: v_add_i32_e64 v24, s[4:5], -1, v24
-; SDAG-NEXT: v_or_b32_e32 v10, v16, v10
-; SDAG-NEXT: v_ashrrev_i32_e32 v16, 31, v12
+; SDAG-NEXT: v_ashrrev_i32_e32 v8, 31, v8
; SDAG-NEXT: v_addc_u32_e64 v27, s[4:5], -1, v27, s[4:5]
-; SDAG-NEXT: v_or_b32_e32 v11, v17, v11
-; SDAG-NEXT: v_and_b32_e32 v12, 1, v16
-; SDAG-NEXT: v_and_b32_e32 v34, v16, v3
-; SDAG-NEXT: v_and_b32_e32 v35, v16, v2
-; SDAG-NEXT: v_and_b32_e32 v17, v16, v23
-; SDAG-NEXT: v_and_b32_e32 v16, v16, v22
+; SDAG-NEXT: v_or_b32_e32 v6, v6, v12
+; SDAG-NEXT: v_or_b32_e32 v11, v9, v11
+; SDAG-NEXT: v_and_b32_e32 v12, 1, v8
+; SDAG-NEXT: v_and_b32_e32 v34, v8, v3
+; SDAG-NEXT: v_and_b32_e32 v35, v8, v2
+; SDAG-NEXT: v_and_b32_e32 v9, v8, v23
+; SDAG-NEXT: v_and_b32_e32 v8, v8, v22
; SDAG-NEXT: v_addc_u32_e64 v28, s[4:5], -1, v28, s[4:5]
-; SDAG-NEXT: v_sub_i32_e32 v14, vcc, v14, v16
+; SDAG-NEXT: v_sub_i32_e32 v14, vcc, v14, v8
; SDAG-NEXT: v_addc_u32_e64 v29, s[4:5], -1, v29, s[4:5]
-; SDAG-NEXT: v_subb_u32_e32 v15, vcc, v15, v17, vcc
-; SDAG-NEXT: v_or_b32_e32 v17, v27, v29
-; SDAG-NEXT: v_or_b32_e32 v16, v24, v28
-; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[16:17]
-; SDAG-NEXT: v_subb_u32_e32 v8, vcc, v8, v35, vcc
+; SDAG-NEXT: v_subb_u32_e32 v15, vcc, v15, v9, vcc
+; SDAG-NEXT: v_or_b32_e32 v9, v27, v29
+; SDAG-NEXT: v_or_b32_e32 v8, v24, v28
+; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[8:9]
+; SDAG-NEXT: v_subb_u32_e32 v16, vcc, v16, v35, vcc
; SDAG-NEXT: v_or_b32_e32 v7, v5, v7
; SDAG-NEXT: v_or_b32_e32 v6, v4, v6
-; SDAG-NEXT: v_subb_u32_e32 v9, vcc, v9, v34, vcc
+; SDAG-NEXT: v_subb_u32_e32 v17, vcc, v17, v34, vcc
; SDAG-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11]
-; SDAG-NEXT: v_mov_b32_e32 v17, v13
-; SDAG-NEXT: v_mov_b32_e32 v16, v12
+; SDAG-NEXT: v_mov_b32_e32 v8, v12
+; SDAG-NEXT: v_mov_b32_e32 v9, v13
; SDAG-NEXT: s_andn2_b64 exec, exec, s[10:11]
; SDAG-NEXT: s_cbranch_execnz .LBB0_9
; SDAG-NEXT: ; %bb.10: ; %Flow
@@ -979,11 +979,11 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_addc_u32_e32 v32, vcc, -1, v10, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v22, v1, v0, s[4:5]
; SDAG-NEXT: v_addc_u32_e32 v33, vcc, -1, v11, vcc
-; SDAG-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-NEXT: v_mov_b32_e32 v1, 0
; SDAG-NEXT: s_mov_b64 s[4:5], 0
; SDAG-NEXT: v_mov_b32_e32 v24, 0
; SDAG-NEXT: v_mov_b32_e32 v25, 0
+; SDAG-NEXT: v_mov_b32_e32 v0, 0
+; SDAG-NEXT: v_mov_b32_e32 v1, 0
; SDAG-NEXT: v_mov_b32_e32 v21, 0
; SDAG-NEXT: .LBB2_3: ; %udiv-do-while3
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -1141,25 +1141,25 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_cndmask_b32_e64 v8, v8, v5, s[4:5]
; SDAG-NEXT: v_lshr_b64 v[5:6], v[6:7], v9
; SDAG-NEXT: v_cndmask_b32_e32 v18, v18, v20, vcc
-; SDAG-NEXT: v_cndmask_b32_e32 v21, 0, v6, vcc
-; SDAG-NEXT: v_cndmask_b32_e32 v20, 0, v5, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v23, 0, v6, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v22, 0, v5, vcc
; SDAG-NEXT: v_add_i32_e32 v6, vcc, -1, v12
; SDAG-NEXT: v_addc_u32_e32 v27, vcc, -1, v13, vcc
; SDAG-NEXT: v_addc_u32_e32 v28, vcc, -1, v14, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v7, v18, v4, s[4:5]
; SDAG-NEXT: v_addc_u32_e32 v29, vcc, -1, v15, vcc
+; SDAG-NEXT: s_mov_b64 s[10:11], 0
+; SDAG-NEXT: v_mov_b32_e32 v20, 0
+; SDAG-NEXT: v_mov_b32_e32 v21, 0
; SDAG-NEXT: v_mov_b32_e32 v4, 0
; SDAG-NEXT: v_mov_b32_e32 v5, 0
-; SDAG-NEXT: s_mov_b64 s[10:11], 0
-; SDAG-NEXT: v_mov_b32_e32 v22, 0
-; SDAG-NEXT: v_mov_b32_e32 v23, 0
; SDAG-NEXT: v_mov_b32_e32 v19, 0
; SDAG-NEXT: .LBB2_9: ; %udiv-do-while
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; SDAG-NEXT: v_lshl_b64 v[20:21], v[20:21], 1
+; SDAG-NEXT: v_lshl_b64 v[22:23], v[22:23], 1
; SDAG-NEXT: v_lshrrev_b32_e32 v18, 31, v8
; SDAG-NEXT: v_lshl_b64 v[7:8], v[7:8], 1
-; SDAG-NEXT: v_or_b32_e32 v20, v20, v18
+; SDAG-NEXT: v_or_b32_e32 v22, v22, v18
; SDAG-NEXT: v_lshrrev_b32_e32 v18, 31, v1
; SDAG-NEXT: v_lshl_b64 v[0:1], v[0:1], 1
; SDAG-NEXT: v_or_b32_e32 v7, v7, v18
@@ -1168,32 +1168,32 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_sub_i32_e32 v18, vcc, v6, v7
; SDAG-NEXT: v_subb_u32_e32 v18, vcc, v27, v8, vcc
; SDAG-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
-; SDAG-NEXT: v_subb_u32_e32 v18, vcc, v28, v20, vcc
-; SDAG-NEXT: v_subb_u32_e32 v18, vcc, v29, v21, vcc
+; SDAG-NEXT: v_subb_u32_e32 v18, vcc, v28, v22, vcc
+; SDAG-NEXT: v_subb_u32_e32 v18, vcc, v29, v23, vcc
; SDAG-NEXT: v_add_i32_e64 v9, s[4:5], -1, v9
-; SDAG-NEXT: v_or_b32_e32 v10, v22, v10
-; SDAG-NEXT: v_ashrrev_i32_e32 v22, 31, v18
+; SDAG-NEXT: v_or_b32_e32 v10, v20, v10
+; SDAG-NEXT: v_ashrrev_i32_e32 v20, 31, v18
; SDAG-NEXT: v_addc_u32_e64 v24, s[4:5], -1, v24, s[4:5]
-; SDAG-NEXT: v_or_b32_e32 v11, v23, v11
-; SDAG-NEXT: v_and_b32_e32 v18, 1, v22
-; SDAG-NEXT: v_and_b32_e32 v30, v22, v15
-; SDAG-NEXT: v_and_b32_e32 v31, v22, v14
-; SDAG-NEXT: v_and_b32_e32 v23, v22, v13
-; SDAG-NEXT: v_and_b32_e32 v22, v22, v12
+; SDAG-NEXT: v_or_b32_e32 v11, v21, v11
+; SDAG-NEXT: v_and_b32_e32 v18, 1, v20
+; SDAG-NEXT: v_and_b32_e32 v30, v20, v15
+; SDAG-NEXT: v_and_b32_e32 v31, v20, v14
+; SDAG-NEXT: v_and_b32_e32 v21, v20, v13
+; SDAG-NEXT: v_and_b32_e32 v20, v20, v12
; SDAG-NEXT: v_addc_u32_e64 v25, s[4:5], -1, v25, s[4:5]
-; SDAG-NEXT: v_sub_i32_e32 v7, vcc, v7, v22
+; SDAG-NEXT: v_sub_i32_e32 v7, vcc, v7, v20
; SDAG-NEXT: v_addc_u32_e64 v26, s[4:5], -1, v26, s[4:5]
-; SDAG-NEXT: v_subb_u32_e32 v8, vcc, v8, v23, vcc
-; SDAG-NEXT: v_or_b32_e32 v23, v24, v26
-; SDAG-NEXT: v_or_b32_e32 v22, v9, v25
-; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[22:23]
-; SDAG-NEXT: v_subb_u32_e32 v20, vcc, v20, v31, vcc
+; SDAG-NEXT: v_subb_u32_e32 v8, vcc, v8, v21, vcc
+; SDAG-NEXT: v_or_b32_e32 v21, v24, v26
+; SDAG-NEXT: v_or_b32_e32 v20, v9, v25
+; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[20:21]
+; SDAG-NEXT: v_subb_u32_e32 v22, vcc, v22, v31, vcc
; SDAG-NEXT: v_or_b32_e32 v1, v5, v1
; SDAG-NEXT: v_or_b32_e32 v0, v4, v0
-; SDAG-NEXT: v_subb_u32_e32 v21, vcc, v21, v30, vcc
+; SDAG-NEXT: v_subb_u32_e32 v23, vcc, v23, v30, vcc
; SDAG-NEXT: s_or_b64 s[10:11], s[4:5], s[10:11]
-; SDAG-NEXT: v_mov_b32_e32 v23, v19
-; SDAG-NEXT: v_mov_b32_e32 v22, v18
+; SDAG-NEXT: v_mov_b32_e32 v21, v19
+; SDAG-NEXT: v_mov_b32_e32 v20, v18
; SDAG-NEXT: s_andn2_b64 exec, exec, s[10:11]
; SDAG-NEXT: s_cbranch_execnz .LBB2_9
; SDAG-NEXT: ; %bb.10: ; %Flow
@@ -1741,11 +1741,11 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_addc_u32_e32 v36, vcc, -1, v29, vcc
; SDAG-NEXT: v_addc_u32_e32 v37, vcc, -1, v8, vcc
; SDAG-NEXT: v_addc_u32_e32 v38, vcc, -1, v9, vcc
-; SDAG-NEXT: v_mov_b32_e32 v20, 0
-; SDAG-NEXT: v_mov_b32_e32 v21, 0
; SDAG-NEXT: s_mov_b64 s[10:11], 0
; SDAG-NEXT: v_mov_b32_e32 v26, 0
; SDAG-NEXT: v_mov_b32_e32 v27, 0
+; SDAG-NEXT: v_mov_b32_e32 v20, 0
+; SDAG-NEXT: v_mov_b32_e32 v21, 0
; SDAG-NEXT: v_mov_b32_e32 v19, 0
; SDAG-NEXT: .LBB4_3: ; %udiv-do-while3
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -1928,11 +1928,11 @@ define <2 x i128> @v_srem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_addc_u32_e32 v49, vcc, -1, v32, vcc
; SDAG-NEXT: v_addc_u32_e32 v50, vcc, -1, v12, vcc
; SDAG-NEXT: v_addc_u32_e32 v51, vcc, -1, v13, vcc
-; SDAG-NEXT: v_mov_b32_e32 v20, 0
-; SDAG-NEXT: v_mov_b32_e32 v21, 0
; SDAG-NEXT: s_mov_b64 s[10:11], 0
; SDAG-NEXT: v_mov_b32_e32 v24, 0
; SDAG-NEXT: v_mov_b32_e32 v25, 0
+; SDAG-NEXT: v_mov_b32_e32 v20, 0
+; SDAG-NEXT: v_mov_b32_e32 v21, 0
; SDAG-NEXT: v_mov_b32_e32 v19, 0
; SDAG-NEXT: .LBB4_9: ; %udiv-do-while
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -2600,11 +2600,11 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_addc_u32_e32 v35, vcc, -1, v9, vcc
; SDAG-NEXT: v_addc_u32_e32 v36, vcc, -1, v10, vcc
; SDAG-NEXT: v_addc_u32_e32 v37, vcc, -1, v11, vcc
-; SDAG-NEXT: v_mov_b32_e32 v22, 0
-; SDAG-NEXT: v_mov_b32_e32 v23, 0
; SDAG-NEXT: s_mov_b64 s[4:5], 0
; SDAG-NEXT: v_mov_b32_e32 v28, 0
; SDAG-NEXT: v_mov_b32_e32 v29, 0
+; SDAG-NEXT: v_mov_b32_e32 v22, 0
+; SDAG-NEXT: v_mov_b32_e32 v23, 0
; SDAG-NEXT: v_mov_b32_e32 v21, 0
; SDAG-NEXT: .LBB5_3: ; %udiv-do-while3
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -2769,11 +2769,11 @@ define <2 x i128> @v_urem_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
; SDAG-NEXT: v_addc_u32_e32 v39, vcc, -1, v13, vcc
; SDAG-NEXT: v_addc_u32_e32 v48, vcc, -1, v14, vcc
; SDAG-NEXT: v_addc_u32_e32 v49, vcc, -1, v15, vcc
-; SDAG-NEXT: v_mov_b32_e32 v24, 0
-; SDAG-NEXT: v_mov_b32_e32 v25, 0
; SDAG-NEXT: s_mov_b64 s[10:11], 0
; SDAG-NEXT: v_mov_b32_e32 v28, 0
; SDAG-NEXT: v_mov_b32_e32 v29, 0
+; SDAG-NEXT: v_mov_b32_e32 v24, 0
+; SDAG-NEXT: v_mov_b32_e32 v25, 0
; SDAG-NEXT: v_mov_b32_e32 v23, 0
; SDAG-NEXT: .LBB5_9: ; %udiv-do-while
; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
diff --git a/llvm/test/CodeGen/AMDGPU/divergent-branch-uniform-condition.ll b/llvm/test/CodeGen/AMDGPU/divergent-branch-uniform-condition.ll
index 230955946445d..6484f644de9a4 100644
--- a/llvm/test/CodeGen/AMDGPU/divergent-branch-uniform-condition.ll
+++ b/llvm/test/CodeGen/AMDGPU/divergent-branch-uniform-condition.ll
@@ -29,8 +29,7 @@ define amdgpu_ps void @main(i32 %0, float %1) {
; ISA-NEXT: s_branch .LBB0_3
; ISA-NEXT: .LBB0_1: ; %Flow1
; ISA-NEXT: ; in Loop: Header=BB0_3 Depth=1
-; ISA-NEXT: s_or_b64 exec, exec, s[4:5]
-; ISA-NEXT: s_mov_b64 s[4:5], s[6:7]
+; ISA-NEXT: s_or_b64 exec, exec, s[6:7]
; ISA-NEXT: s_mov_b64 s[6:7], 0
; ISA-NEXT: .LBB0_2: ; %Flow
; ISA-NEXT: ; in Loop: Header=BB0_3 Depth=1
@@ -40,32 +39,30 @@ define amdgpu_ps void @main(i32 %0, float %1) {
; ISA-NEXT: s_or_b64 s[0:1], s[10:11], s[0:1]
; ISA-NEXT: s_or_b64 s[2:3], s[2:3], s[6:7]
; ISA-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; ISA-NEXT: s_cbranch_execz .LBB0_7
+; ISA-NEXT: s_cbranch_execz .LBB0_6
; ISA-NEXT: .LBB0_3: ; %loop
; ISA-NEXT: ; =>This Inner Loop Header: Depth=1
; ISA-NEXT: s_or_b64 s[4:5], s[4:5], exec
; ISA-NEXT: s_cmp_lt_u32 s8, 32
; ISA-NEXT: s_mov_b64 s[6:7], -1
-; ISA-NEXT: s_cbranch_scc0 .LBB0_6
+; ISA-NEXT: s_cbranch_scc0 .LBB0_2
; ISA-NEXT: ; %bb.4: ; %endif1
; ISA-NEXT: ; in Loop: Header=BB0_3 Depth=1
-; ISA-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; ISA-NEXT: s_mov_b64 s[4:5], -1
+; ISA-NEXT: s_and_saveexec_b64 s[6:7], vcc
; ISA-NEXT: s_cbranch_execz .LBB0_1
; ISA-NEXT: ; %bb.5: ; %endif2
; ISA-NEXT: ; in Loop: Header=BB0_3 Depth=1
; ISA-NEXT: s_add_i32 s8, s8, 1
-; ISA-NEXT: s_xor_b64 s[6:7], exec, -1
+; ISA-NEXT: s_xor_b64 s[4:5], exec, -1
; ISA-NEXT: s_branch .LBB0_1
-; ISA-NEXT: .LBB0_6: ; in Loop: Header=BB0_3 Depth=1
-; ISA-NEXT: ; implicit-def: $sgpr8
-; ISA-NEXT: s_branch .LBB0_2
-; ISA-NEXT: .LBB0_7: ; %Flow2
+; ISA-NEXT: .LBB0_6: ; %Flow2
; ISA-NEXT: s_or_b64 exec, exec, s[0:1]
; ISA-NEXT: v_mov_b32_e32 v1, 0
; ISA-NEXT: s_and_saveexec_b64 s[0:1], s[2:3]
-; ISA-NEXT: ; %bb.8: ; %if1
+; ISA-NEXT: ; %bb.7: ; %if1
; ISA-NEXT: v_sqrt_f32_e32 v1, v0
-; ISA-NEXT: ; %bb.9: ; %endloop
+; ISA-NEXT: ; %bb.8: ; %endloop
; ISA-NEXT: s_or_b64 exec, exec, s[0:1]
; ISA-NEXT: exp mrt0, v1, v1, v1, v1 done vm
; ISA-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/expand-mov-b64-globaladdr.ll b/llvm/test/CodeGen/AMDGPU/expand-mov-b64-globaladdr.ll
index 6316bd2c2d450..69666747f4a94 100644
--- a/llvm/test/CodeGen/AMDGPU/expand-mov-b64-globaladdr.ll
+++ b/llvm/test/CodeGen/AMDGPU/expand-mov-b64-globaladdr.ll
@@ -19,8 +19,10 @@ define amdgpu_ps ptr addrspace(4) @v_mov_b64_pseudo_globaladdr(i1 %cond) {
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 1, v2
; GFX10-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX10-NEXT: ; %bb.1: ; %other
-; GFX10-NEXT: v_mov_b32_e32 v0, gv at abs32@lo
-; GFX10-NEXT: v_mov_b32_e32 v1, gv at abs32@hi
+; GFX10-NEXT: s_mov_b32 s3, gv at abs32@hi
+; GFX10-NEXT: s_mov_b32 s2, gv at abs32@lo
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: ; %bb.2: ; %join
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
@@ -35,11 +37,13 @@ define amdgpu_ps ptr addrspace(4) @v_mov_b64_pseudo_globaladdr(i1 %cond) {
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_cmpx_ne_u32_e32 1, v2
; GFX11-NEXT: ; %bb.1: ; %other
-; GFX11-NEXT: v_mov_b32_e32 v0, gv at abs32@lo
-; GFX11-NEXT: v_mov_b32_e32 v1, gv at abs32@hi
+; GFX11-NEXT: s_mov_b32 s3, gv at abs32@hi
+; GFX11-NEXT: s_mov_b32 s2, gv at abs32@lo
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
; GFX11-NEXT: ; %bb.2: ; %join
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-NEXT: v_readfirstlane_b32 s1, v1
; GFX11-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-nondeterminism.ll b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-nondeterminism.ll
index 093575ad076e5..2d12a414c41e5 100644
--- a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-nondeterminism.ll
+++ b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-nondeterminism.ll
@@ -4,6 +4,7 @@
define amdgpu_gs void @f(i32 inreg %arg, i32 %arg1, i32 %arg2) {
; CHECK-LABEL: f:
; CHECK: ; %bb.0: ; %bb
+; CHECK-NEXT: v_mov_b32_e32 v3, 0
; CHECK-NEXT: s_cmp_eq_u32 s0, 0
; CHECK-NEXT: s_mov_b32 s0, 0
; CHECK-NEXT: s_cbranch_scc1 .LBB0_2
@@ -11,18 +12,16 @@ define amdgpu_gs void @f(i32 inreg %arg, i32 %arg1, i32 %arg2) {
; CHECK-NEXT: v_mov_b32_e32 v2, v0
; CHECK-NEXT: s_branch .LBB0_3
; CHECK-NEXT: .LBB0_2:
-; CHECK-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, 1
+; CHECK-NEXT: v_dual_mov_b32 v2, 1 :: v_dual_mov_b32 v1, 0
; CHECK-NEXT: .LBB0_3: ; %bb4
-; CHECK-NEXT: v_mov_b32_e32 v3, 0
+; CHECK-NEXT: v_mov_b32_e32 v4, v3
+; CHECK-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v0, 0
; CHECK-NEXT: s_mov_b32 s1, s0
; CHECK-NEXT: s_mov_b32 s2, s0
; CHECK-NEXT: s_mov_b32 s3, s0
-; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; CHECK-NEXT: v_mov_b32_e32 v4, v3
-; CHECK-NEXT: v_mov_b32_e32 v5, v3
-; CHECK-NEXT: buffer_store_b128 v[2:5], v3, s[0:3], 0 idxen
+; CHECK-NEXT: buffer_store_b128 v[2:5], v0, s[0:3], 0 idxen
; CHECK-NEXT: v_mov_b32_e32 v2, v3
-; CHECK-NEXT: buffer_store_b128 v[1:4], v3, s[0:3], 0 idxen
+; CHECK-NEXT: buffer_store_b128 v[1:4], v0, s[0:3], 0 idxen
; CHECK-NEXT: s_endpgm
bb:
%i = icmp eq i32 %arg, 0
diff --git a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-phi-regression-issue130646-issue130119.ll b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-phi-regression-issue130646-issue130119.ll
index bdca3b99b108b..62c6d8204d780 100644
--- a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-phi-regression-issue130646-issue130119.ll
+++ b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-phi-regression-issue130646-issue130119.ll
@@ -33,13 +33,11 @@ define double @issue130646(i64 %arg) {
; CHECK-NEXT: .LBB0_3: ; %for.body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_cmp_eq_u64 s[4:5], 0
-; CHECK-NEXT: v_readfirstlane_b32 s8, v0
-; CHECK-NEXT: v_readfirstlane_b32 s9, v1
; CHECK-NEXT: s_cbranch_scc0 .LBB0_1
; CHECK-NEXT: ; %bb.4: ; in Loop: Header=BB0_3 Depth=1
; CHECK-NEXT: s_mov_b64 s[6:7], -1
; CHECK-NEXT: ; implicit-def: $vgpr2_vgpr3
-; CHECK-NEXT: s_mov_b64 s[4:5], s[8:9]
+; CHECK-NEXT: ; implicit-def: $sgpr4_sgpr5
; CHECK-NEXT: s_branch .LBB0_2
; CHECK-NEXT: .LBB0_5: ; %for.cond.cleanup
; CHECK-NEXT: v_mov_b32_e32 v0, 0
diff --git a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-sgpr32-to-vgpr16.ll b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-sgpr32-to-vgpr16.ll
index 109127df9a533..32c4d7d49a54b 100644
--- a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-sgpr32-to-vgpr16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-sgpr32-to-vgpr16.ll
@@ -32,17 +32,17 @@ define i32 @test_cvt_pk_fp8_f32(float %x, float %y) {
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE killed [[V_CVT_PK_FP8_F32_t16_e64_]], %subreg.lo16, killed [[COPY6]], %subreg.hi16
; CHECK-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 7
; CHECK-NEXT: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 killed [[REG_SEQUENCE]], killed [[S_MOV_B32_2]], implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ ; CHECK-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 1
; CHECK-NEXT: [[SI_IF:%[0-9]+]]:sreg_32 = SI_IF killed [[V_CMP_NE_U32_e64_]], %bb.2, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.a2:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
+ ; CHECK-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.UnifiedReturnBlock:
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[V_MOV_B32_e32_1]], %bb.0, [[V_MOV_B32_e32_2]], %bb.1
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[S_MOV_B32_3]], %bb.0, [[S_MOV_B32_4]], %bb.1
; CHECK-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; CHECK-NEXT: $vgpr0 = COPY [[PHI]]
; CHECK-NEXT: SI_RETURN implicit $vgpr0
@@ -90,17 +90,17 @@ define i32 @test_cvt_pk_fp8_f32_e5m3(float %x, float %y) {
; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE killed [[V_CVT_PK_FP8_F32_gfx1250_t16_e64_]], %subreg.lo16, killed [[COPY6]], %subreg.hi16
; CHECK-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 7
; CHECK-NEXT: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_NE_U32_e64 killed [[REG_SEQUENCE]], killed [[S_MOV_B32_2]], implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ ; CHECK-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 1
; CHECK-NEXT: [[SI_IF:%[0-9]+]]:sreg_32 = SI_IF killed [[V_CMP_NE_U32_e64_]], %bb.2, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.a2:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2, implicit $exec
+ ; CHECK-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.UnifiedReturnBlock:
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[V_MOV_B32_e32_1]], %bb.0, [[V_MOV_B32_e32_2]], %bb.1
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[S_MOV_B32_3]], %bb.0, [[S_MOV_B32_4]], %bb.1
; CHECK-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; CHECK-NEXT: $vgpr0 = COPY [[PHI]]
; CHECK-NEXT: SI_RETURN implicit $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-rtn.ll b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-rtn.ll
index f8cd7cdbe1cec..7b649b3ff2f15 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomic-fadd.f32-rtn.ll
@@ -46,7 +46,7 @@ define amdgpu_ps float @global_atomic_fadd_f32_saddr_rtn_atomicrmw(ptr addrspace
; GFX90A-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
; GFX90A-NEXT: [[COPY3:%[0-9]+]]:sreg_64_xexec_xnull = COPY [[REG_SEQUENCE]]
; GFX90A-NEXT: [[SI_PS_LIVE:%[0-9]+]]:sreg_64 = SI_PS_LIVE
- ; GFX90A-NEXT: [[DEF:%[0-9]+]]:av_32 = IMPLICIT_DEF
+ ; GFX90A-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
; GFX90A-NEXT: [[SI_IF:%[0-9]+]]:sreg_64 = SI_IF killed [[SI_PS_LIVE]], %bb.3, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX90A-NEXT: S_BRANCH %bb.1
; GFX90A-NEXT: {{ $}}
@@ -82,7 +82,7 @@ define amdgpu_ps float @global_atomic_fadd_f32_saddr_rtn_atomicrmw(ptr addrspace
; GFX90A-NEXT: early-clobber %2:sgpr_32 = STRICT_WWM killed [[V_READLANE_B32_]], implicit $exec
; GFX90A-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 killed [[V_MBCNT_HI_U32_B32_e64_]], [[S_MOV_B32_]], implicit $exec
; GFX90A-NEXT: [[COPY9:%[0-9]+]]:vreg_1 = COPY [[V_CMP_EQ_U32_e64_]]
- ; GFX90A-NEXT: [[DEF2:%[0-9]+]]:av_32 = IMPLICIT_DEF
+ ; GFX90A-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
; GFX90A-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64 = SI_IF [[V_CMP_EQ_U32_e64_]], %bb.4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX90A-NEXT: S_BRANCH %bb.2
; GFX90A-NEXT: {{ $}}
@@ -131,7 +131,7 @@ define amdgpu_ps float @global_atomic_fadd_f32_saddr_rtn_atomicrmw(ptr addrspace
; GFX942-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
; GFX942-NEXT: [[COPY3:%[0-9]+]]:sreg_64_xexec_xnull = COPY [[REG_SEQUENCE]]
; GFX942-NEXT: [[SI_PS_LIVE:%[0-9]+]]:sreg_64 = SI_PS_LIVE
- ; GFX942-NEXT: [[DEF:%[0-9]+]]:av_32 = IMPLICIT_DEF
+ ; GFX942-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
; GFX942-NEXT: [[SI_IF:%[0-9]+]]:sreg_64 = SI_IF killed [[SI_PS_LIVE]], %bb.3, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX942-NEXT: S_BRANCH %bb.1
; GFX942-NEXT: {{ $}}
@@ -167,7 +167,7 @@ define amdgpu_ps float @global_atomic_fadd_f32_saddr_rtn_atomicrmw(ptr addrspace
; GFX942-NEXT: early-clobber %2:sgpr_32 = STRICT_WWM killed [[V_READLANE_B32_]], implicit $exec
; GFX942-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 killed [[V_MBCNT_HI_U32_B32_e64_]], [[S_MOV_B32_]], implicit $exec
; GFX942-NEXT: [[COPY9:%[0-9]+]]:vreg_1 = COPY [[V_CMP_EQ_U32_e64_]]
- ; GFX942-NEXT: [[DEF2:%[0-9]+]]:av_32 = IMPLICIT_DEF
+ ; GFX942-NEXT: [[DEF2:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
; GFX942-NEXT: [[SI_IF1:%[0-9]+]]:sreg_64 = SI_IF [[V_CMP_EQ_U32_e64_]], %bb.4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX942-NEXT: S_BRANCH %bb.2
; GFX942-NEXT: {{ $}}
@@ -216,7 +216,7 @@ define amdgpu_ps float @global_atomic_fadd_f32_saddr_rtn_atomicrmw(ptr addrspace
; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
; GFX11-NEXT: [[COPY3:%[0-9]+]]:sreg_64_xexec_xnull = COPY [[REG_SEQUENCE]]
; GFX11-NEXT: [[SI_PS_LIVE:%[0-9]+]]:sreg_32 = SI_PS_LIVE
- ; GFX11-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GFX11-NEXT: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
; GFX11-NEXT: [[SI_IF:%[0-9]+]]:sreg_32 = SI_IF killed [[SI_PS_LIVE]], %bb.3, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX11-NEXT: S_BRANCH %bb.1
; GFX11-NEXT: {{ $}}
@@ -253,7 +253,7 @@ define amdgpu_ps float @global_atomic_fadd_f32_saddr_rtn_atomicrmw(ptr addrspace
; GFX11-NEXT: early-clobber %2:sgpr_32 = STRICT_WWM killed [[V_READLANE_B32_1]], implicit $exec
; GFX11-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32 = V_CMP_EQ_U32_e64 killed [[V_MBCNT_LO_U32_B32_e64_]], [[S_MOV_B32_]], implicit $exec
; GFX11-NEXT: [[COPY6:%[0-9]+]]:vreg_1 = COPY [[V_CMP_EQ_U32_e64_]]
- ; GFX11-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; GFX11-NEXT: [[DEF3:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
; GFX11-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32 = SI_IF [[V_CMP_EQ_U32_e64_]], %bb.4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; GFX11-NEXT: S_BRANCH %bb.2
; GFX11-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/implicit-def-muse.ll b/llvm/test/CodeGen/AMDGPU/implicit-def-muse.ll
index bd56a25e33e4e..a381477155456 100644
--- a/llvm/test/CodeGen/AMDGPU/implicit-def-muse.ll
+++ b/llvm/test/CodeGen/AMDGPU/implicit-def-muse.ll
@@ -2,9 +2,9 @@
; RUN: llc -mtriple=amdgpu6.00 -stop-after=amdgpu-isel -enable-new-pm -o - %s | FileCheck %s
; CHECK-LABEL: vcopy_i1_undef
-; CHECK: [[IMPDEF0:%[0-9]+]]:vreg_1 = IMPLICIT_DEF
+; CHECK: [[IMPDEF0:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
; CHECK-NOT: COPY
-; CHECK: [[IMPDEF1:%[0-9]+]]:vreg_1 = IMPLICIT_DEF
+; CHECK: [[IMPDEF1:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
; CHECK-NOT: COPY [[IMPDEF0]]
; CHECK-NOT: COPY [[IMPDEF1]]
; CHECK: .false:
diff --git a/llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll b/llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll
index 02802330fe785..f7ddd4dfdd428 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll
@@ -19,13 +19,13 @@ define amdgpu_kernel void @widget(ptr addrspace(1) %arg, i1 %arg1) #0 {
; CHECK-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr8_sgpr9, 0, 0 :: (dereferenceable invariant load (s64) from %ir.arg.kernarg.offset1, align 16, addrspace 4)
; CHECK-NEXT: $sgpr20 = S_ADD_U32 $sgpr20, killed $sgpr17, implicit-def $scc, implicit-def $sgpr20_sgpr21_sgpr22_sgpr23
; CHECK-NEXT: $sgpr21 = S_ADDC_U32 $sgpr21, 0, implicit-def dead $scc, implicit killed $scc, implicit-def $sgpr20_sgpr21_sgpr22_sgpr23
- ; CHECK-NEXT: renamable $vgpr1 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr4_sgpr5 = S_MOV_B64 0
; CHECK-NEXT: S_WAITCNT .Lgkmcnt_0
; CHECK-NEXT: S_BITCMP1_B32 killed renamable $sgpr2, 0, implicit-def $scc
- ; CHECK-NEXT: renamable $sgpr4_sgpr5 = S_MOV_B64 0
- ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr1 = V_MOV_B32_e32 0, implicit $exec
; CHECK-NEXT: renamable $sgpr2_sgpr3 = S_CSELECT_B64 -1, 0, implicit killed $scc
; CHECK-NEXT: $vgpr2_vgpr3 = V_PK_MOV_B32 8, 0, 8, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; CHECK-NEXT: renamable $sgpr6_sgpr7 = IMPLICIT_DEF
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
index 6cc43c0785048..56f278de6ee49 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
@@ -407,8 +407,9 @@ define amdgpu_ps void @non_cst_non_compare_input(ptr addrspace(1) %out, i32 %tid
; GFX10-NEXT: s_and_saveexec_b32 s1, vcc_lo
; GFX10-NEXT: s_xor_b32 s1, exec_lo, s1
; GFX10-NEXT: ; %bb.1: ; %B
-; GFX10-NEXT: v_cmp_gt_u32_e64 s0, 2, v2
+; GFX10-NEXT: v_cmp_gt_u32_e32 vcc_lo, 2, v2
; GFX10-NEXT: ; implicit-def: $vgpr2
+; GFX10-NEXT: s_and_b32 s0, vcc_lo, exec_lo
; GFX10-NEXT: ; %bb.2: ; %Flow
; GFX10-NEXT: s_andn2_saveexec_b32 s1, s1
; GFX10-NEXT: ; %bb.3: ; %A
@@ -431,8 +432,9 @@ define amdgpu_ps void @non_cst_non_compare_input(ptr addrspace(1) %out, i32 %tid
; GFX11-NEXT: v_cmpx_ne_u32_e32 0, v3
; GFX11-NEXT: s_xor_b32 s1, exec_lo, s1
; GFX11-NEXT: ; %bb.1: ; %B
-; GFX11-NEXT: v_cmp_gt_u32_e64 s0, 2, v2
+; GFX11-NEXT: v_cmp_gt_u32_e32 vcc_lo, 2, v2
; GFX11-NEXT: ; implicit-def: $vgpr2
+; GFX11-NEXT: s_and_b32 s0, vcc_lo, exec_lo
; GFX11-NEXT: ; %bb.2: ; %Flow
; GFX11-NEXT: s_and_not1_saveexec_b32 s1, s1
; GFX11-NEXT: ; %bb.3: ; %A
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.ll
index 68da5a9866184..1568aa0a800cb 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.ll
@@ -410,7 +410,8 @@ define amdgpu_ps void @non_cst_non_compare_input(ptr addrspace(1) %out, i32 %tid
; CHECK-NEXT: s_and_saveexec_b64 s[2:3], vcc
; CHECK-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; CHECK-NEXT: ; %bb.1: ; %B
-; CHECK-NEXT: v_cmp_gt_u32_e64 s[0:1], 2, v2
+; CHECK-NEXT: v_cmp_gt_u32_e32 vcc, 2, v2
+; CHECK-NEXT: s_and_b64 s[0:1], vcc, exec
; CHECK-NEXT: ; implicit-def: $vgpr2
; CHECK-NEXT: ; %bb.2: ; %Flow
; CHECK-NEXT: s_andn2_saveexec_b64 s[2:3], s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/loop_break.ll b/llvm/test/CodeGen/AMDGPU/loop_break.ll
index a63df79e1d1b4..3e21570d1b4bc 100644
--- a/llvm/test/CodeGen/AMDGPU/loop_break.ll
+++ b/llvm/test/CodeGen/AMDGPU/loop_break.ll
@@ -115,24 +115,28 @@ define amdgpu_kernel void @undef_phi_cond_break_loop(i32 %arg) #0 {
; GCN-NEXT: s_load_dword s3, s[4:5], 0x9
; GCN-NEXT: s_mov_b64 s[0:1], 0
; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: ; implicit-def: $sgpr4_sgpr5
; GCN-NEXT: ; implicit-def: $sgpr6
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_subrev_i32_e32 v0, vcc, s3, v0
; GCN-NEXT: s_mov_b32 s3, 0xf000
; GCN-NEXT: .LBB1_1: ; %bb1
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
+; GCN-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
; GCN-NEXT: s_cmp_gt_i32 s6, -1
-; GCN-NEXT: ; implicit-def: $sgpr4_sgpr5
; GCN-NEXT: s_cbranch_scc1 .LBB1_3
; GCN-NEXT: ; %bb.2: ; %bb4
; GCN-NEXT: ; in Loop: Header=BB1_1 Depth=1
; GCN-NEXT: buffer_load_dword v1, off, s[0:3], 0 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_cmp_ge_i32_e64 s[4:5], v0, v1
+; GCN-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
+; GCN-NEXT: v_cmp_ge_i32_e32 vcc, v0, v1
+; GCN-NEXT: s_and_b64 s[8:9], vcc, exec
+; GCN-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
; GCN-NEXT: .LBB1_3: ; %Flow
; GCN-NEXT: ; in Loop: Header=BB1_1 Depth=1
-; GCN-NEXT: s_and_b64 s[4:5], exec, s[4:5]
-; GCN-NEXT: s_or_b64 s[0:1], s[4:5], s[0:1]
+; GCN-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GCN-NEXT: s_or_b64 s[0:1], s[8:9], s[0:1]
; GCN-NEXT: s_add_i32 s6, s6, 1
; GCN-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN-NEXT: s_cbranch_execnz .LBB1_1
diff --git a/llvm/test/CodeGen/AMDGPU/machine-sink-loop-var-out-of-divergent-loop-swdev407790.ll b/llvm/test/CodeGen/AMDGPU/machine-sink-loop-var-out-of-divergent-loop-swdev407790.ll
index 3fb1166c70006..2ebf9b1821b19 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-sink-loop-var-out-of-divergent-loop-swdev407790.ll
+++ b/llvm/test/CodeGen/AMDGPU/machine-sink-loop-var-out-of-divergent-loop-swdev407790.ll
@@ -12,6 +12,7 @@ define void @machinesink_loop_variable_out_of_divergent_loop(i32 %arg, i1 %cmp49
; CHECK-NEXT: v_and_b32_e32 v3, 1, v3
; CHECK-NEXT: s_mov_b32 s6, 0
; CHECK-NEXT: v_cmp_ne_u32_e64 s4, 1, v1
+; CHECK-NEXT: v_mov_b32_e32 v1, 0
; CHECK-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v3
; CHECK-NEXT: s_inst_prefetch 0x1
; CHECK-NEXT: s_branch .LBB0_3
@@ -22,7 +23,7 @@ define void @machinesink_loop_variable_out_of_divergent_loop(i32 %arg, i1 %cmp49
; CHECK-NEXT: .LBB0_2: ; %Flow1
; CHECK-NEXT: ; in Loop: Header=BB0_3 Depth=1
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s7
-; CHECK-NEXT: v_cmp_ne_u32_e64 s5, 0, v3
+; CHECK-NEXT: v_cmp_ne_u32_e64 s5, 0, v4
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; j lastloop entry
; CHECK-NEXT: ;;#ASMEND
@@ -32,13 +33,13 @@ define void @machinesink_loop_variable_out_of_divergent_loop(i32 %arg, i1 %cmp49
; CHECK-NEXT: .LBB0_3: ; %for.body33
; CHECK-NEXT: ; =>This Loop Header: Depth=1
; CHECK-NEXT: ; Child Loop BB0_6 Depth 2
-; CHECK-NEXT: v_mov_b32_e32 v1, 0
+; CHECK-NEXT: v_mov_b32_e32 v4, 0
; CHECK-NEXT: v_mov_b32_e32 v3, 0
; CHECK-NEXT: s_and_saveexec_b32 s7, s4
; CHECK-NEXT: s_cbranch_execz .LBB0_2
; CHECK-NEXT: ; %bb.4: ; %for.body51.preheader
; CHECK-NEXT: ; in Loop: Header=BB0_3 Depth=1
-; CHECK-NEXT: v_mov_b32_e32 v4, v2
+; CHECK-NEXT: v_mov_b32_e32 v5, v2
; CHECK-NEXT: s_mov_b32 s9, 4
; CHECK-NEXT: s_mov_b32 s8, 0
; CHECK-NEXT: s_branch .LBB0_6
@@ -46,27 +47,27 @@ define void @machinesink_loop_variable_out_of_divergent_loop(i32 %arg, i1 %cmp49
; CHECK-NEXT: .LBB0_5: ; %if.end118
; CHECK-NEXT: ; in Loop: Header=BB0_6 Depth=2
; CHECK-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; CHECK-NEXT: v_add_nc_u32_e32 v4, 4, v1
+; CHECK-NEXT: v_add_nc_u32_e32 v5, 4, v3
; CHECK-NEXT: s_add_i32 s9, s9, 4
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ; backedge
; CHECK-NEXT: ;;#ASMEND
-; CHECK-NEXT: v_cmp_ge_u32_e64 s5, v4, v0
+; CHECK-NEXT: v_cmp_ge_u32_e64 s5, v5, v0
; CHECK-NEXT: s_or_b32 s8, s5, s8
; CHECK-NEXT: s_andn2_b32 exec_lo, exec_lo, s8
; CHECK-NEXT: s_cbranch_execz .LBB0_1
; CHECK-NEXT: .LBB0_6: ; %for.body51
; CHECK-NEXT: ; Parent Loop BB0_3 Depth=1
; CHECK-NEXT: ; => This Inner Loop Header: Depth=2
-; CHECK-NEXT: v_mov_b32_e32 v1, v4
-; CHECK-NEXT: v_mov_b32_e32 v3, 1
+; CHECK-NEXT: v_mov_b32_e32 v3, v5
+; CHECK-NEXT: v_mov_b32_e32 v4, 1
; CHECK-NEXT: s_and_saveexec_b32 s5, vcc_lo
; CHECK-NEXT: s_cbranch_execz .LBB0_5
; CHECK-NEXT: ; %bb.7: ; %if.then112
; CHECK-NEXT: ; in Loop: Header=BB0_6 Depth=2
-; CHECK-NEXT: v_mov_b32_e32 v3, 0
-; CHECK-NEXT: v_mov_b32_e32 v4, s9
-; CHECK-NEXT: ds_write_b32 v3, v4
+; CHECK-NEXT: v_mov_b32_e32 v5, s9
+; CHECK-NEXT: v_mov_b32_e32 v4, 0
+; CHECK-NEXT: ds_write_b32 v1, v5
; CHECK-NEXT: s_branch .LBB0_5
; CHECK-NEXT: .LBB0_8: ; %for.body159.preheader
; CHECK-NEXT: s_inst_prefetch 0x2
diff --git a/llvm/test/CodeGen/AMDGPU/mad_int24.ll b/llvm/test/CodeGen/AMDGPU/mad_int24.ll
index d621fbe5a2051..b0c5675e430fe 100644
--- a/llvm/test/CodeGen/AMDGPU/mad_int24.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad_int24.ll
@@ -164,8 +164,8 @@ define void @mad24_destroyed_knownbits_2(i32 %arg, i32 %arg1, i32 %arg2, ptr add
; GCN-LABEL: mad24_destroyed_knownbits_2:
; GCN: ; %bb.0: ; %bb
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v5, 1
; GCN-NEXT: s_mov_b64 s[4:5], 0
+; GCN-NEXT: v_mov_b32_e32 v5, 1
; GCN-NEXT: .LBB3_1: ; %bb6
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN-NEXT: v_mad_i32_i24 v0, v0, v5, v5
@@ -191,8 +191,8 @@ define void @mad24_destroyed_knownbits_2(i32 %arg, i32 %arg1, i32 %arg2, ptr add
; VI-LABEL: mad24_destroyed_knownbits_2:
; VI: ; %bb.0: ; %bb
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v5, 1
; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: v_mov_b32_e32 v5, 1
; VI-NEXT: .LBB3_1: ; %bb6
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_mad_i32_i24 v0, v0, v5, v5
diff --git a/llvm/test/CodeGen/AMDGPU/masked-load-vectortypes.ll b/llvm/test/CodeGen/AMDGPU/masked-load-vectortypes.ll
index d2fff2eb3629e..954e7f56fb7e0 100644
--- a/llvm/test/CodeGen/AMDGPU/masked-load-vectortypes.ll
+++ b/llvm/test/CodeGen/AMDGPU/masked-load-vectortypes.ll
@@ -5,13 +5,15 @@ define <2 x i32> @uniform_masked_load_ptr1_mask_v2i32(ptr addrspace(1) inreg noc
; GFX942-LABEL: uniform_masked_load_ptr1_mask_v2i32:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_mov_b32 s2, 0
; GFX942-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX942-NEXT: s_mov_b32 s3, s2
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[2:3]
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB0_2
; GFX942-NEXT: ; %bb.1: ; %cond.load
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: global_load_dwordx2 v[0:1], v0, s[0:1]
; GFX942-NEXT: .LBB0_2:
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
@@ -28,15 +30,18 @@ define <4 x i32> @uniform_masked_load_ptr1_mask_v4i32(ptr addrspace(1) inreg noc
; GFX942-LABEL: uniform_masked_load_ptr1_mask_v4i32:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_mov_b32 s4, 0
; GFX942-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX942-NEXT: s_mov_b32 s5, s4
+; GFX942-NEXT: s_mov_b32 s6, s4
+; GFX942-NEXT: s_mov_b32 s7, s4
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
+; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB1_2
; GFX942-NEXT: ; %bb.1: ; %cond.load
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: global_load_dwordx4 v[0:3], v0, s[0:1]
; GFX942-NEXT: .LBB1_2:
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
@@ -53,15 +58,18 @@ define <4 x float> @uniform_masked_load_ptr1_mask_v4f32(ptr addrspace(1) inreg n
; GFX942-LABEL: uniform_masked_load_ptr1_mask_v4f32:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_mov_b32 s4, 0
; GFX942-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX942-NEXT: s_mov_b32 s5, s4
+; GFX942-NEXT: s_mov_b32 s6, s4
+; GFX942-NEXT: s_mov_b32 s7, s4
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
+; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB2_2
; GFX942-NEXT: ; %bb.1: ; %cond.load
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: global_load_dwordx4 v[0:3], v0, s[0:1]
; GFX942-NEXT: .LBB2_2:
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
@@ -78,22 +86,26 @@ define <8 x i32> @uniform_masked_load_ptr1_mask_v8i32(ptr addrspace(1) inreg noc
; GFX942-LABEL: uniform_masked_load_ptr1_mask_v8i32:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_mov_b32 s4, 0
; GFX942-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX942-NEXT: s_mov_b32 s5, s4
+; GFX942-NEXT: s_mov_b32 s6, s4
+; GFX942-NEXT: s_mov_b32 s7, s4
+; GFX942-NEXT: s_mov_b32 s8, s4
+; GFX942-NEXT: s_mov_b32 s9, s4
+; GFX942-NEXT: s_mov_b32 s10, s4
+; GFX942-NEXT: s_mov_b32 s11, s4
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: v_mov_b32_e32 v4, v0
-; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_mov_b32_e32 v6, v0
-; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
+; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], s[8:9]
+; GFX942-NEXT: v_mov_b64_e32 v[6:7], s[10:11]
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB3_2
; GFX942-NEXT: ; %bb.1: ; %cond.load
-; GFX942-NEXT: global_load_dwordx4 v[4:7], v0, s[0:1] offset:16
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: global_load_dwordx4 v[0:3], v0, s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-NEXT: global_load_dwordx4 v[4:7], v8, s[0:1] offset:16
+; GFX942-NEXT: global_load_dwordx4 v[0:3], v8, s[0:1]
; GFX942-NEXT: .LBB3_2:
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -109,22 +121,26 @@ define <8 x float> @uniform_masked_load_ptr1_mask_v8f32(ptr addrspace(1) inreg n
; GFX942-LABEL: uniform_masked_load_ptr1_mask_v8f32:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_mov_b32 s4, 0
; GFX942-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX942-NEXT: s_mov_b32 s5, s4
+; GFX942-NEXT: s_mov_b32 s6, s4
+; GFX942-NEXT: s_mov_b32 s7, s4
+; GFX942-NEXT: s_mov_b32 s8, s4
+; GFX942-NEXT: s_mov_b32 s9, s4
+; GFX942-NEXT: s_mov_b32 s10, s4
+; GFX942-NEXT: s_mov_b32 s11, s4
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: v_mov_b32_e32 v4, v0
-; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_mov_b32_e32 v6, v0
-; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
+; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], s[8:9]
+; GFX942-NEXT: v_mov_b64_e32 v[6:7], s[10:11]
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB4_2
; GFX942-NEXT: ; %bb.1: ; %cond.load
-; GFX942-NEXT: global_load_dwordx4 v[4:7], v0, s[0:1] offset:16
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: global_load_dwordx4 v[0:3], v0, s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-NEXT: global_load_dwordx4 v[4:7], v8, s[0:1] offset:16
+; GFX942-NEXT: global_load_dwordx4 v[0:3], v8, s[0:1]
; GFX942-NEXT: .LBB4_2:
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -140,15 +156,18 @@ define <8 x i16> @uniform_masked_load_ptr1_mask_v8i16(ptr addrspace(1) inreg noc
; GFX942-LABEL: uniform_masked_load_ptr1_mask_v8i16:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_mov_b32 s4, 0
; GFX942-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX942-NEXT: s_mov_b32 s5, s4
+; GFX942-NEXT: s_mov_b32 s6, s4
+; GFX942-NEXT: s_mov_b32 s7, s4
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
+; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB5_2
; GFX942-NEXT: ; %bb.1: ; %cond.load
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: global_load_dwordx4 v[0:3], v0, s[0:1]
; GFX942-NEXT: .LBB5_2:
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
@@ -165,15 +184,18 @@ define <8 x half> @uniform_masked_load_ptr1_mask_v8f16(ptr addrspace(1) inreg no
; GFX942-LABEL: uniform_masked_load_ptr1_mask_v8f16:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_mov_b32 s4, 0
; GFX942-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX942-NEXT: s_mov_b32 s5, s4
+; GFX942-NEXT: s_mov_b32 s6, s4
+; GFX942-NEXT: s_mov_b32 s7, s4
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
+; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB6_2
; GFX942-NEXT: ; %bb.1: ; %cond.load
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: global_load_dwordx4 v[0:3], v0, s[0:1]
; GFX942-NEXT: .LBB6_2:
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
@@ -190,15 +212,18 @@ define <8 x bfloat> @uniform_masked_load_ptr1_mask_v8bf16(ptr addrspace(1) inreg
; GFX942-LABEL: uniform_masked_load_ptr1_mask_v8bf16:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_mov_b32 s4, 0
; GFX942-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX942-NEXT: s_mov_b32 s5, s4
+; GFX942-NEXT: s_mov_b32 s6, s4
+; GFX942-NEXT: s_mov_b32 s7, s4
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
+; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB7_2
; GFX942-NEXT: ; %bb.1: ; %cond.load
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: global_load_dwordx4 v[0:3], v0, s[0:1]
; GFX942-NEXT: .LBB7_2:
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
@@ -236,7 +261,8 @@ define <16 x i8> @uniform_masked_load_ptr1_mask_v16i8(ptr addrspace(1) inreg noc
; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB8_2
; GFX942-NEXT: ; %bb.1: ; %cond.load
-; GFX942-NEXT: global_load_dwordx4 v[16:19], v16, s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: global_load_dwordx4 v[16:19], v0, s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_lshrrev_b32_e32 v15, 24, v19
; GFX942-NEXT: v_lshrrev_b32_e32 v14, 16, v19
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-loop.ll b/llvm/test/CodeGen/AMDGPU/mfma-loop.ll
index 995d0c3178d35..f189fce3ca229 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-loop.ll
@@ -10,38 +10,38 @@
define amdgpu_kernel void @test_mfma_loop_zeroinit(ptr addrspace(1) %arg) #0 {
; GFX908-LABEL: test_mfma_loop_zeroinit:
; GFX908: ; %bb.0: ; %entry
-; GFX908-NEXT: v_accvgpr_write_b32 a31, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a30, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a29, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a28, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a27, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a26, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a25, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a24, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a23, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a22, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a21, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a20, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a19, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a18, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a17, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a16, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a15, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a14, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a13, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a12, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a11, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a10, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a9, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a8, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a7, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a6, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a5, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a4, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a3, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a2, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a1, 0
; GFX908-NEXT: v_accvgpr_write_b32 a0, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a1, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a2, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a3, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a4, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a5, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a6, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a7, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a8, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a9, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a10, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a11, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a12, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a13, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a14, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a15, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a16, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a17, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a18, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a19, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a20, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a21, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a22, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a23, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a24, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a25, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a26, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a27, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a28, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a29, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a30, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a31, 0
; GFX908-NEXT: s_mov_b32 s0, 16
; GFX908-NEXT: v_mov_b32_e32 v0, 2.0
; GFX908-NEXT: v_mov_b32_e32 v1, 1.0
@@ -101,41 +101,41 @@ define amdgpu_kernel void @test_mfma_loop_zeroinit(ptr addrspace(1) %arg) #0 {
;
; GFX90A-LABEL: test_mfma_loop_zeroinit:
; GFX90A: ; %bb.0: ; %entry
-; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_mov_b32 s0, 16
+; GFX90A-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-NEXT: v_mov_b32_e32 v1, 0
+; GFX90A-NEXT: v_mov_b32_e32 v2, 0
+; GFX90A-NEXT: v_mov_b32_e32 v3, 0
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
+; GFX90A-NEXT: v_mov_b32_e32 v7, 0
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-NEXT: v_mov_b32_e32 v9, 0
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
+; GFX90A-NEXT: v_mov_b32_e32 v11, 0
+; GFX90A-NEXT: v_mov_b32_e32 v12, 0
+; GFX90A-NEXT: v_mov_b32_e32 v13, 0
+; GFX90A-NEXT: v_mov_b32_e32 v14, 0
+; GFX90A-NEXT: v_mov_b32_e32 v15, 0
+; GFX90A-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-NEXT: v_mov_b32_e32 v17, 0
+; GFX90A-NEXT: v_mov_b32_e32 v18, 0
+; GFX90A-NEXT: v_mov_b32_e32 v19, 0
+; GFX90A-NEXT: v_mov_b32_e32 v20, 0
+; GFX90A-NEXT: v_mov_b32_e32 v21, 0
+; GFX90A-NEXT: v_mov_b32_e32 v22, 0
+; GFX90A-NEXT: v_mov_b32_e32 v23, 0
+; GFX90A-NEXT: v_mov_b32_e32 v24, 0
+; GFX90A-NEXT: v_mov_b32_e32 v25, 0
+; GFX90A-NEXT: v_mov_b32_e32 v26, 0
+; GFX90A-NEXT: v_mov_b32_e32 v27, 0
+; GFX90A-NEXT: v_mov_b32_e32 v28, 0
+; GFX90A-NEXT: v_mov_b32_e32 v29, 0
+; GFX90A-NEXT: v_mov_b32_e32 v30, 0
+; GFX90A-NEXT: v_mov_b32_e32 v31, 0
; GFX90A-NEXT: v_mov_b32_e32 v32, 2.0
; GFX90A-NEXT: v_mov_b32_e32 v33, 1.0
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: v_mov_b32_e32 v4, v0
-; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_mov_b32_e32 v6, v0
-; GFX90A-NEXT: v_mov_b32_e32 v7, v0
-; GFX90A-NEXT: v_mov_b32_e32 v8, v0
-; GFX90A-NEXT: v_mov_b32_e32 v9, v0
-; GFX90A-NEXT: v_mov_b32_e32 v10, v0
-; GFX90A-NEXT: v_mov_b32_e32 v11, v0
-; GFX90A-NEXT: v_mov_b32_e32 v12, v0
-; GFX90A-NEXT: v_mov_b32_e32 v13, v0
-; GFX90A-NEXT: v_mov_b32_e32 v14, v0
-; GFX90A-NEXT: v_mov_b32_e32 v15, v0
-; GFX90A-NEXT: v_mov_b32_e32 v16, v0
-; GFX90A-NEXT: v_mov_b32_e32 v17, v0
-; GFX90A-NEXT: v_mov_b32_e32 v18, v0
-; GFX90A-NEXT: v_mov_b32_e32 v19, v0
-; GFX90A-NEXT: v_mov_b32_e32 v20, v0
-; GFX90A-NEXT: v_mov_b32_e32 v21, v0
-; GFX90A-NEXT: v_mov_b32_e32 v22, v0
-; GFX90A-NEXT: v_mov_b32_e32 v23, v0
-; GFX90A-NEXT: v_mov_b32_e32 v24, v0
-; GFX90A-NEXT: v_mov_b32_e32 v25, v0
-; GFX90A-NEXT: v_mov_b32_e32 v26, v0
-; GFX90A-NEXT: v_mov_b32_e32 v27, v0
-; GFX90A-NEXT: v_mov_b32_e32 v28, v0
-; GFX90A-NEXT: v_mov_b32_e32 v29, v0
-; GFX90A-NEXT: v_mov_b32_e32 v30, v0
-; GFX90A-NEXT: v_mov_b32_e32 v31, v0
; GFX90A-NEXT: .LBB0_1: ; %for.cond.preheader
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_nop 1
@@ -160,41 +160,41 @@ define amdgpu_kernel void @test_mfma_loop_zeroinit(ptr addrspace(1) %arg) #0 {
;
; GFX942-LABEL: test_mfma_loop_zeroinit:
; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_mov_b32 s0, 16
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-NEXT: v_mov_b32_e32 v3, 0
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-NEXT: v_mov_b32_e32 v5, 0
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v7, 0
+; GFX942-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-NEXT: v_mov_b32_e32 v9, 0
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
+; GFX942-NEXT: v_mov_b32_e32 v11, 0
+; GFX942-NEXT: v_mov_b32_e32 v12, 0
+; GFX942-NEXT: v_mov_b32_e32 v13, 0
+; GFX942-NEXT: v_mov_b32_e32 v14, 0
+; GFX942-NEXT: v_mov_b32_e32 v15, 0
+; GFX942-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-NEXT: v_mov_b32_e32 v17, 0
+; GFX942-NEXT: v_mov_b32_e32 v18, 0
+; GFX942-NEXT: v_mov_b32_e32 v19, 0
+; GFX942-NEXT: v_mov_b32_e32 v20, 0
+; GFX942-NEXT: v_mov_b32_e32 v21, 0
+; GFX942-NEXT: v_mov_b32_e32 v22, 0
+; GFX942-NEXT: v_mov_b32_e32 v23, 0
+; GFX942-NEXT: v_mov_b32_e32 v24, 0
+; GFX942-NEXT: v_mov_b32_e32 v25, 0
+; GFX942-NEXT: v_mov_b32_e32 v26, 0
+; GFX942-NEXT: v_mov_b32_e32 v27, 0
+; GFX942-NEXT: v_mov_b32_e32 v28, 0
+; GFX942-NEXT: v_mov_b32_e32 v29, 0
+; GFX942-NEXT: v_mov_b32_e32 v30, 0
+; GFX942-NEXT: v_mov_b32_e32 v31, 0
; GFX942-NEXT: v_mov_b32_e32 v32, 2.0
; GFX942-NEXT: v_mov_b32_e32 v33, 1.0
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: v_mov_b32_e32 v4, v0
-; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_mov_b32_e32 v6, v0
-; GFX942-NEXT: v_mov_b32_e32 v7, v0
-; GFX942-NEXT: v_mov_b32_e32 v8, v0
-; GFX942-NEXT: v_mov_b32_e32 v9, v0
-; GFX942-NEXT: v_mov_b32_e32 v10, v0
-; GFX942-NEXT: v_mov_b32_e32 v11, v0
-; GFX942-NEXT: v_mov_b32_e32 v12, v0
-; GFX942-NEXT: v_mov_b32_e32 v13, v0
-; GFX942-NEXT: v_mov_b32_e32 v14, v0
-; GFX942-NEXT: v_mov_b32_e32 v15, v0
-; GFX942-NEXT: v_mov_b32_e32 v16, v0
-; GFX942-NEXT: v_mov_b32_e32 v17, v0
-; GFX942-NEXT: v_mov_b32_e32 v18, v0
-; GFX942-NEXT: v_mov_b32_e32 v19, v0
-; GFX942-NEXT: v_mov_b32_e32 v20, v0
-; GFX942-NEXT: v_mov_b32_e32 v21, v0
-; GFX942-NEXT: v_mov_b32_e32 v22, v0
-; GFX942-NEXT: v_mov_b32_e32 v23, v0
-; GFX942-NEXT: v_mov_b32_e32 v24, v0
-; GFX942-NEXT: v_mov_b32_e32 v25, v0
-; GFX942-NEXT: v_mov_b32_e32 v26, v0
-; GFX942-NEXT: v_mov_b32_e32 v27, v0
-; GFX942-NEXT: v_mov_b32_e32 v28, v0
-; GFX942-NEXT: v_mov_b32_e32 v29, v0
-; GFX942-NEXT: v_mov_b32_e32 v30, v0
-; GFX942-NEXT: v_mov_b32_e32 v31, v0
; GFX942-NEXT: .LBB0_1: ; %for.cond.preheader
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_nop 1
@@ -332,41 +332,42 @@ define amdgpu_kernel void @test_mfma_loop_unfoldable_splat(ptr addrspace(1) %arg
;
; GFX90A-LABEL: test_mfma_loop_unfoldable_splat:
; GFX90A: ; %bb.0: ; %entry
-; GFX90A-NEXT: v_mov_b32_e32 v0, 0x42f60000
+; GFX90A-NEXT: s_mov_b32 s1, 0x42f60000
; GFX90A-NEXT: s_mov_b32 s0, 16
+; GFX90A-NEXT: v_mov_b32_e32 v0, s1
; GFX90A-NEXT: v_mov_b32_e32 v32, 2.0
; GFX90A-NEXT: v_mov_b32_e32 v33, 1.0
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: v_mov_b32_e32 v4, v0
-; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_mov_b32_e32 v6, v0
-; GFX90A-NEXT: v_mov_b32_e32 v7, v0
-; GFX90A-NEXT: v_mov_b32_e32 v8, v0
-; GFX90A-NEXT: v_mov_b32_e32 v9, v0
-; GFX90A-NEXT: v_mov_b32_e32 v10, v0
-; GFX90A-NEXT: v_mov_b32_e32 v11, v0
-; GFX90A-NEXT: v_mov_b32_e32 v12, v0
-; GFX90A-NEXT: v_mov_b32_e32 v13, v0
-; GFX90A-NEXT: v_mov_b32_e32 v14, v0
-; GFX90A-NEXT: v_mov_b32_e32 v15, v0
-; GFX90A-NEXT: v_mov_b32_e32 v16, v0
-; GFX90A-NEXT: v_mov_b32_e32 v17, v0
-; GFX90A-NEXT: v_mov_b32_e32 v18, v0
-; GFX90A-NEXT: v_mov_b32_e32 v19, v0
-; GFX90A-NEXT: v_mov_b32_e32 v20, v0
-; GFX90A-NEXT: v_mov_b32_e32 v21, v0
-; GFX90A-NEXT: v_mov_b32_e32 v22, v0
-; GFX90A-NEXT: v_mov_b32_e32 v23, v0
-; GFX90A-NEXT: v_mov_b32_e32 v24, v0
-; GFX90A-NEXT: v_mov_b32_e32 v25, v0
-; GFX90A-NEXT: v_mov_b32_e32 v26, v0
-; GFX90A-NEXT: v_mov_b32_e32 v27, v0
-; GFX90A-NEXT: v_mov_b32_e32 v28, v0
-; GFX90A-NEXT: v_mov_b32_e32 v29, v0
-; GFX90A-NEXT: v_mov_b32_e32 v30, v0
-; GFX90A-NEXT: v_mov_b32_e32 v31, v0
+; GFX90A-NEXT: v_mov_b32_e32 v1, s1
+; GFX90A-NEXT: v_mov_b32_e32 v2, s1
+; GFX90A-NEXT: v_mov_b32_e32 v3, s1
+; GFX90A-NEXT: v_mov_b32_e32 v4, s1
+; GFX90A-NEXT: v_mov_b32_e32 v5, s1
+; GFX90A-NEXT: v_mov_b32_e32 v6, s1
+; GFX90A-NEXT: v_mov_b32_e32 v7, s1
+; GFX90A-NEXT: v_mov_b32_e32 v8, s1
+; GFX90A-NEXT: v_mov_b32_e32 v9, s1
+; GFX90A-NEXT: v_mov_b32_e32 v10, s1
+; GFX90A-NEXT: v_mov_b32_e32 v11, s1
+; GFX90A-NEXT: v_mov_b32_e32 v12, s1
+; GFX90A-NEXT: v_mov_b32_e32 v13, s1
+; GFX90A-NEXT: v_mov_b32_e32 v14, s1
+; GFX90A-NEXT: v_mov_b32_e32 v15, s1
+; GFX90A-NEXT: v_mov_b32_e32 v16, s1
+; GFX90A-NEXT: v_mov_b32_e32 v17, s1
+; GFX90A-NEXT: v_mov_b32_e32 v18, s1
+; GFX90A-NEXT: v_mov_b32_e32 v19, s1
+; GFX90A-NEXT: v_mov_b32_e32 v20, s1
+; GFX90A-NEXT: v_mov_b32_e32 v21, s1
+; GFX90A-NEXT: v_mov_b32_e32 v22, s1
+; GFX90A-NEXT: v_mov_b32_e32 v23, s1
+; GFX90A-NEXT: v_mov_b32_e32 v24, s1
+; GFX90A-NEXT: v_mov_b32_e32 v25, s1
+; GFX90A-NEXT: v_mov_b32_e32 v26, s1
+; GFX90A-NEXT: v_mov_b32_e32 v27, s1
+; GFX90A-NEXT: v_mov_b32_e32 v28, s1
+; GFX90A-NEXT: v_mov_b32_e32 v29, s1
+; GFX90A-NEXT: v_mov_b32_e32 v30, s1
+; GFX90A-NEXT: v_mov_b32_e32 v31, s1
; GFX90A-NEXT: .LBB1_1: ; %for.cond.preheader
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_nop 1
@@ -391,70 +392,71 @@ define amdgpu_kernel void @test_mfma_loop_unfoldable_splat(ptr addrspace(1) %arg
;
; GFX942-LABEL: test_mfma_loop_unfoldable_splat:
; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: v_mov_b32_e32 v0, 0x42f60000
+; GFX942-NEXT: s_mov_b32 s1, 0x42f60000
; GFX942-NEXT: s_mov_b32 s0, 16
+; GFX942-NEXT: v_mov_b32_e32 v0, s1
; GFX942-NEXT: v_mov_b32_e32 v32, 2.0
; GFX942-NEXT: v_mov_b32_e32 v33, 1.0
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: v_mov_b32_e32 v4, v0
-; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_mov_b32_e32 v6, v0
-; GFX942-NEXT: v_mov_b32_e32 v7, v0
-; GFX942-NEXT: v_mov_b32_e32 v8, v0
-; GFX942-NEXT: v_mov_b32_e32 v9, v0
-; GFX942-NEXT: v_mov_b32_e32 v10, v0
-; GFX942-NEXT: v_mov_b32_e32 v11, v0
-; GFX942-NEXT: v_mov_b32_e32 v12, v0
-; GFX942-NEXT: v_mov_b32_e32 v13, v0
-; GFX942-NEXT: v_mov_b32_e32 v14, v0
-; GFX942-NEXT: v_mov_b32_e32 v15, v0
-; GFX942-NEXT: v_mov_b32_e32 v16, v0
-; GFX942-NEXT: v_mov_b32_e32 v17, v0
-; GFX942-NEXT: v_mov_b32_e32 v18, v0
-; GFX942-NEXT: v_mov_b32_e32 v19, v0
-; GFX942-NEXT: v_mov_b32_e32 v20, v0
-; GFX942-NEXT: v_mov_b32_e32 v21, v0
-; GFX942-NEXT: v_mov_b32_e32 v22, v0
-; GFX942-NEXT: v_mov_b32_e32 v23, v0
-; GFX942-NEXT: v_mov_b32_e32 v24, v0
-; GFX942-NEXT: v_mov_b32_e32 v25, v0
-; GFX942-NEXT: v_mov_b32_e32 v26, v0
-; GFX942-NEXT: v_mov_b32_e32 v27, v0
-; GFX942-NEXT: v_mov_b32_e32 v28, v0
-; GFX942-NEXT: v_mov_b32_e32 v29, v0
-; GFX942-NEXT: v_mov_b32_e32 v30, v0
-; GFX942-NEXT: v_mov_b32_e32 v31, v0
-; GFX942-NEXT: .LBB1_1: ; %for.cond.preheader
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_nop 1
-; GFX942-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v33, v32, v[0:31]
-; GFX942-NEXT: s_add_i32 s0, s0, -1
-; GFX942-NEXT: s_cmp_lg_u32 s0, 0
-; GFX942-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX942-NEXT: ; %bb.2: ; %exit
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-NEXT: v_mov_b32_e32 v32, 0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: s_nop 11
-; GFX942-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
-; GFX942-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
-; GFX942-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
-; GFX942-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
-; GFX942-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
-; GFX942-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
-; GFX942-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
-; GFX942-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
-; GFX942-NEXT: s_endpgm
-entry:
- br label %for.cond.preheader
-
-for.cond.preheader:
- %phi = phi <32 x float> [ <float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0>, %entry ], [ %mai.1, %for.cond.preheader ]
- %c = phi i32 [ 0, %entry ], [ %inc, %for.cond.preheader ]
- %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %phi, i32 0, i32 0, i32 0)
- %inc = add nuw nsw i32 %c, 1
+; GFX942-NEXT: v_mov_b32_e32 v1, s1
+; GFX942-NEXT: v_mov_b32_e32 v2, s1
+; GFX942-NEXT: v_mov_b32_e32 v3, s1
+; GFX942-NEXT: v_mov_b32_e32 v4, s1
+; GFX942-NEXT: v_mov_b32_e32 v5, s1
+; GFX942-NEXT: v_mov_b32_e32 v6, s1
+; GFX942-NEXT: v_mov_b32_e32 v7, s1
+; GFX942-NEXT: v_mov_b32_e32 v8, s1
+; GFX942-NEXT: v_mov_b32_e32 v9, s1
+; GFX942-NEXT: v_mov_b32_e32 v10, s1
+; GFX942-NEXT: v_mov_b32_e32 v11, s1
+; GFX942-NEXT: v_mov_b32_e32 v12, s1
+; GFX942-NEXT: v_mov_b32_e32 v13, s1
+; GFX942-NEXT: v_mov_b32_e32 v14, s1
+; GFX942-NEXT: v_mov_b32_e32 v15, s1
+; GFX942-NEXT: v_mov_b32_e32 v16, s1
+; GFX942-NEXT: v_mov_b32_e32 v17, s1
+; GFX942-NEXT: v_mov_b32_e32 v18, s1
+; GFX942-NEXT: v_mov_b32_e32 v19, s1
+; GFX942-NEXT: v_mov_b32_e32 v20, s1
+; GFX942-NEXT: v_mov_b32_e32 v21, s1
+; GFX942-NEXT: v_mov_b32_e32 v22, s1
+; GFX942-NEXT: v_mov_b32_e32 v23, s1
+; GFX942-NEXT: v_mov_b32_e32 v24, s1
+; GFX942-NEXT: v_mov_b32_e32 v25, s1
+; GFX942-NEXT: v_mov_b32_e32 v26, s1
+; GFX942-NEXT: v_mov_b32_e32 v27, s1
+; GFX942-NEXT: v_mov_b32_e32 v28, s1
+; GFX942-NEXT: v_mov_b32_e32 v29, s1
+; GFX942-NEXT: v_mov_b32_e32 v30, s1
+; GFX942-NEXT: v_mov_b32_e32 v31, s1
+; GFX942-NEXT: .LBB1_1: ; %for.cond.preheader
+; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-NEXT: s_nop 1
+; GFX942-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v33, v32, v[0:31]
+; GFX942-NEXT: s_add_i32 s0, s0, -1
+; GFX942-NEXT: s_cmp_lg_u32 s0, 0
+; GFX942-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX942-NEXT: ; %bb.2: ; %exit
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: v_mov_b32_e32 v32, 0
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_nop 11
+; GFX942-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX942-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; GFX942-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; GFX942-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; GFX942-NEXT: global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; GFX942-NEXT: global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; GFX942-NEXT: global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; GFX942-NEXT: global_store_dwordx4 v32, v[0:3], s[0:1]
+; GFX942-NEXT: s_endpgm
+entry:
+ br label %for.cond.preheader
+
+for.cond.preheader:
+ %phi = phi <32 x float> [ <float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0, float 123.0>, %entry ], [ %mai.1, %for.cond.preheader ]
+ %c = phi i32 [ 0, %entry ], [ %inc, %for.cond.preheader ]
+ %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %phi, i32 0, i32 0, i32 0)
+ %inc = add nuw nsw i32 %c, 1
%cc = icmp eq i32 %inc, 16
br i1 %cc, label %exit, label %for.cond.preheader
@@ -466,40 +468,40 @@ exit:
define amdgpu_kernel void @test_mfma_loop_non_splat(ptr addrspace(1) %arg) #0 {
; GFX908-LABEL: test_mfma_loop_non_splat:
; GFX908: ; %bb.0: ; %entry
+; GFX908-NEXT: v_accvgpr_write_b32 a0, 0
; GFX908-NEXT: v_accvgpr_write_b32 a1, 1.0
-; GFX908-NEXT: v_accvgpr_write_b32 a31, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a30, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a29, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a28, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a27, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a26, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a25, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a24, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a23, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a22, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a21, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a20, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a19, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a18, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a17, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a16, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a15, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a14, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a13, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a12, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a11, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a10, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a9, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a8, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a7, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a6, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a5, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a4, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a3, 0
; GFX908-NEXT: v_accvgpr_write_b32 a2, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a0, 0
-; GFX908-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX908-NEXT: v_accvgpr_write_b32 a3, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a4, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a5, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a6, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a7, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a8, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a9, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a10, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a11, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a12, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a13, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a14, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a15, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a16, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a17, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a18, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a19, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a20, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a21, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a22, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a23, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a24, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a25, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a26, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a27, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a28, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a29, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a30, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a31, 0
; GFX908-NEXT: s_mov_b32 s0, 16
+; GFX908-NEXT: v_mov_b32_e32 v0, 1.0
; GFX908-NEXT: v_mov_b32_e32 v1, 2.0
; GFX908-NEXT: .LBB2_1: ; %for.cond.preheader
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -558,40 +560,40 @@ define amdgpu_kernel void @test_mfma_loop_non_splat(ptr addrspace(1) %arg) #0 {
; GFX90A-LABEL: test_mfma_loop_non_splat:
; GFX90A: ; %bb.0: ; %entry
; GFX90A-NEXT: v_mov_b32_e32 v32, 1.0
-; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_mov_b32 s0, 16
+; GFX90A-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-NEXT: v_mov_b32_e32 v2, 0
+; GFX90A-NEXT: v_mov_b32_e32 v3, 0
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
+; GFX90A-NEXT: v_mov_b32_e32 v7, 0
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-NEXT: v_mov_b32_e32 v9, 0
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
+; GFX90A-NEXT: v_mov_b32_e32 v11, 0
+; GFX90A-NEXT: v_mov_b32_e32 v12, 0
+; GFX90A-NEXT: v_mov_b32_e32 v13, 0
+; GFX90A-NEXT: v_mov_b32_e32 v14, 0
+; GFX90A-NEXT: v_mov_b32_e32 v15, 0
+; GFX90A-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-NEXT: v_mov_b32_e32 v17, 0
+; GFX90A-NEXT: v_mov_b32_e32 v18, 0
+; GFX90A-NEXT: v_mov_b32_e32 v19, 0
+; GFX90A-NEXT: v_mov_b32_e32 v20, 0
+; GFX90A-NEXT: v_mov_b32_e32 v21, 0
+; GFX90A-NEXT: v_mov_b32_e32 v22, 0
+; GFX90A-NEXT: v_mov_b32_e32 v23, 0
+; GFX90A-NEXT: v_mov_b32_e32 v24, 0
+; GFX90A-NEXT: v_mov_b32_e32 v25, 0
+; GFX90A-NEXT: v_mov_b32_e32 v26, 0
+; GFX90A-NEXT: v_mov_b32_e32 v27, 0
+; GFX90A-NEXT: v_mov_b32_e32 v28, 0
+; GFX90A-NEXT: v_mov_b32_e32 v29, 0
+; GFX90A-NEXT: v_mov_b32_e32 v30, 0
+; GFX90A-NEXT: v_mov_b32_e32 v31, 0
; GFX90A-NEXT: v_mov_b32_e32 v33, 2.0
; GFX90A-NEXT: v_mov_b32_e32 v1, v32
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: v_mov_b32_e32 v4, v0
-; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_mov_b32_e32 v6, v0
-; GFX90A-NEXT: v_mov_b32_e32 v7, v0
-; GFX90A-NEXT: v_mov_b32_e32 v8, v0
-; GFX90A-NEXT: v_mov_b32_e32 v9, v0
-; GFX90A-NEXT: v_mov_b32_e32 v10, v0
-; GFX90A-NEXT: v_mov_b32_e32 v11, v0
-; GFX90A-NEXT: v_mov_b32_e32 v12, v0
-; GFX90A-NEXT: v_mov_b32_e32 v13, v0
-; GFX90A-NEXT: v_mov_b32_e32 v14, v0
-; GFX90A-NEXT: v_mov_b32_e32 v15, v0
-; GFX90A-NEXT: v_mov_b32_e32 v16, v0
-; GFX90A-NEXT: v_mov_b32_e32 v17, v0
-; GFX90A-NEXT: v_mov_b32_e32 v18, v0
-; GFX90A-NEXT: v_mov_b32_e32 v19, v0
-; GFX90A-NEXT: v_mov_b32_e32 v20, v0
-; GFX90A-NEXT: v_mov_b32_e32 v21, v0
-; GFX90A-NEXT: v_mov_b32_e32 v22, v0
-; GFX90A-NEXT: v_mov_b32_e32 v23, v0
-; GFX90A-NEXT: v_mov_b32_e32 v24, v0
-; GFX90A-NEXT: v_mov_b32_e32 v25, v0
-; GFX90A-NEXT: v_mov_b32_e32 v26, v0
-; GFX90A-NEXT: v_mov_b32_e32 v27, v0
-; GFX90A-NEXT: v_mov_b32_e32 v28, v0
-; GFX90A-NEXT: v_mov_b32_e32 v29, v0
-; GFX90A-NEXT: v_mov_b32_e32 v30, v0
-; GFX90A-NEXT: v_mov_b32_e32 v31, v0
; GFX90A-NEXT: .LBB2_1: ; %for.cond.preheader
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_nop 1
@@ -617,40 +619,40 @@ define amdgpu_kernel void @test_mfma_loop_non_splat(ptr addrspace(1) %arg) #0 {
; GFX942-LABEL: test_mfma_loop_non_splat:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: v_mov_b32_e32 v32, 1.0
-; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_mov_b32 s0, 16
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-NEXT: v_mov_b32_e32 v3, 0
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-NEXT: v_mov_b32_e32 v5, 0
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v7, 0
+; GFX942-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-NEXT: v_mov_b32_e32 v9, 0
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
+; GFX942-NEXT: v_mov_b32_e32 v11, 0
+; GFX942-NEXT: v_mov_b32_e32 v12, 0
+; GFX942-NEXT: v_mov_b32_e32 v13, 0
+; GFX942-NEXT: v_mov_b32_e32 v14, 0
+; GFX942-NEXT: v_mov_b32_e32 v15, 0
+; GFX942-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-NEXT: v_mov_b32_e32 v17, 0
+; GFX942-NEXT: v_mov_b32_e32 v18, 0
+; GFX942-NEXT: v_mov_b32_e32 v19, 0
+; GFX942-NEXT: v_mov_b32_e32 v20, 0
+; GFX942-NEXT: v_mov_b32_e32 v21, 0
+; GFX942-NEXT: v_mov_b32_e32 v22, 0
+; GFX942-NEXT: v_mov_b32_e32 v23, 0
+; GFX942-NEXT: v_mov_b32_e32 v24, 0
+; GFX942-NEXT: v_mov_b32_e32 v25, 0
+; GFX942-NEXT: v_mov_b32_e32 v26, 0
+; GFX942-NEXT: v_mov_b32_e32 v27, 0
+; GFX942-NEXT: v_mov_b32_e32 v28, 0
+; GFX942-NEXT: v_mov_b32_e32 v29, 0
+; GFX942-NEXT: v_mov_b32_e32 v30, 0
+; GFX942-NEXT: v_mov_b32_e32 v31, 0
; GFX942-NEXT: v_mov_b32_e32 v33, 2.0
; GFX942-NEXT: v_mov_b32_e32 v1, v32
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: v_mov_b32_e32 v4, v0
-; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_mov_b32_e32 v6, v0
-; GFX942-NEXT: v_mov_b32_e32 v7, v0
-; GFX942-NEXT: v_mov_b32_e32 v8, v0
-; GFX942-NEXT: v_mov_b32_e32 v9, v0
-; GFX942-NEXT: v_mov_b32_e32 v10, v0
-; GFX942-NEXT: v_mov_b32_e32 v11, v0
-; GFX942-NEXT: v_mov_b32_e32 v12, v0
-; GFX942-NEXT: v_mov_b32_e32 v13, v0
-; GFX942-NEXT: v_mov_b32_e32 v14, v0
-; GFX942-NEXT: v_mov_b32_e32 v15, v0
-; GFX942-NEXT: v_mov_b32_e32 v16, v0
-; GFX942-NEXT: v_mov_b32_e32 v17, v0
-; GFX942-NEXT: v_mov_b32_e32 v18, v0
-; GFX942-NEXT: v_mov_b32_e32 v19, v0
-; GFX942-NEXT: v_mov_b32_e32 v20, v0
-; GFX942-NEXT: v_mov_b32_e32 v21, v0
-; GFX942-NEXT: v_mov_b32_e32 v22, v0
-; GFX942-NEXT: v_mov_b32_e32 v23, v0
-; GFX942-NEXT: v_mov_b32_e32 v24, v0
-; GFX942-NEXT: v_mov_b32_e32 v25, v0
-; GFX942-NEXT: v_mov_b32_e32 v26, v0
-; GFX942-NEXT: v_mov_b32_e32 v27, v0
-; GFX942-NEXT: v_mov_b32_e32 v28, v0
-; GFX942-NEXT: v_mov_b32_e32 v29, v0
-; GFX942-NEXT: v_mov_b32_e32 v30, v0
-; GFX942-NEXT: v_mov_b32_e32 v31, v0
; GFX942-NEXT: .LBB2_1: ; %for.cond.preheader
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_nop 1
@@ -696,70 +698,70 @@ exit:
define amdgpu_kernel void @test_mfma_loop_unfoldable_seq(ptr addrspace(1) %arg) #0 {
; GFX908-LABEL: test_mfma_loop_unfoldable_seq:
; GFX908: ; %bb.0: ; %entry
-; GFX908-NEXT: v_mov_b32_e32 v0, 0x431a0000
-; GFX908-NEXT: v_mov_b32_e32 v1, 0x43190000
-; GFX908-NEXT: v_mov_b32_e32 v2, 0x43160000
-; GFX908-NEXT: v_accvgpr_write_b32 a31, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a30, v1
-; GFX908-NEXT: v_mov_b32_e32 v0, 0x43180000
-; GFX908-NEXT: v_mov_b32_e32 v1, 0x43170000
-; GFX908-NEXT: v_accvgpr_write_b32 a27, v2
-; GFX908-NEXT: v_accvgpr_write_b32 a29, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a28, v1
-; GFX908-NEXT: v_mov_b32_e32 v0, 0x43150000
-; GFX908-NEXT: v_mov_b32_e32 v1, 0x43140000
-; GFX908-NEXT: v_mov_b32_e32 v2, 0x43130000
-; GFX908-NEXT: v_accvgpr_write_b32 a26, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a25, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a24, v2
-; GFX908-NEXT: v_mov_b32_e32 v0, 0x43120000
-; GFX908-NEXT: v_mov_b32_e32 v1, 0x43110000
-; GFX908-NEXT: v_mov_b32_e32 v2, 0x43100000
-; GFX908-NEXT: v_accvgpr_write_b32 a23, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a22, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a21, v2
-; GFX908-NEXT: v_mov_b32_e32 v0, 0x430f0000
-; GFX908-NEXT: v_mov_b32_e32 v1, 0x430e0000
-; GFX908-NEXT: v_mov_b32_e32 v2, 0x430d0000
-; GFX908-NEXT: v_accvgpr_write_b32 a20, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a19, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a18, v2
-; GFX908-NEXT: v_mov_b32_e32 v0, 0x430c0000
-; GFX908-NEXT: v_mov_b32_e32 v1, 0x430b0000
-; GFX908-NEXT: v_mov_b32_e32 v2, 0x430a0000
-; GFX908-NEXT: v_accvgpr_write_b32 a17, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a16, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a15, v2
-; GFX908-NEXT: v_mov_b32_e32 v0, 0x43090000
-; GFX908-NEXT: v_mov_b32_e32 v1, 0x43080000
-; GFX908-NEXT: v_mov_b32_e32 v2, 0x43070000
-; GFX908-NEXT: v_accvgpr_write_b32 a14, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a13, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a12, v2
-; GFX908-NEXT: v_mov_b32_e32 v0, 0x43060000
-; GFX908-NEXT: v_mov_b32_e32 v1, 0x43050000
-; GFX908-NEXT: v_mov_b32_e32 v2, 0x43040000
-; GFX908-NEXT: v_accvgpr_write_b32 a11, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a10, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a9, v2
-; GFX908-NEXT: v_mov_b32_e32 v0, 0x43030000
-; GFX908-NEXT: v_mov_b32_e32 v1, 0x43020000
-; GFX908-NEXT: v_mov_b32_e32 v2, 0x43010000
-; GFX908-NEXT: v_accvgpr_write_b32 a8, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a7, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a6, v2
-; GFX908-NEXT: v_mov_b32_e32 v0, 0x43000000
-; GFX908-NEXT: v_mov_b32_e32 v1, 0x42fe0000
-; GFX908-NEXT: v_mov_b32_e32 v2, 0x42fc0000
-; GFX908-NEXT: v_accvgpr_write_b32 a5, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a4, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a3, v2
-; GFX908-NEXT: v_mov_b32_e32 v0, 0x42fa0000
+; GFX908-NEXT: v_mov_b32_e32 v0, 0x42f60000
; GFX908-NEXT: v_mov_b32_e32 v1, 0x42f80000
-; GFX908-NEXT: v_mov_b32_e32 v2, 0x42f60000
-; GFX908-NEXT: v_accvgpr_write_b32 a2, v0
+; GFX908-NEXT: v_mov_b32_e32 v2, 0x42fe0000
+; GFX908-NEXT: v_accvgpr_write_b32 a0, v0
; GFX908-NEXT: v_accvgpr_write_b32 a1, v1
-; GFX908-NEXT: v_accvgpr_write_b32 a0, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, 0x42fa0000
+; GFX908-NEXT: v_mov_b32_e32 v1, 0x42fc0000
+; GFX908-NEXT: v_accvgpr_write_b32 a4, v2
+; GFX908-NEXT: v_accvgpr_write_b32 a2, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a3, v1
+; GFX908-NEXT: v_mov_b32_e32 v0, 0x43000000
+; GFX908-NEXT: v_mov_b32_e32 v1, 0x43010000
+; GFX908-NEXT: v_mov_b32_e32 v2, 0x43020000
+; GFX908-NEXT: v_accvgpr_write_b32 a5, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a6, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a7, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, 0x43030000
+; GFX908-NEXT: v_mov_b32_e32 v1, 0x43040000
+; GFX908-NEXT: v_mov_b32_e32 v2, 0x43050000
+; GFX908-NEXT: v_accvgpr_write_b32 a8, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a9, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a10, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, 0x43060000
+; GFX908-NEXT: v_mov_b32_e32 v1, 0x43070000
+; GFX908-NEXT: v_mov_b32_e32 v2, 0x43080000
+; GFX908-NEXT: v_accvgpr_write_b32 a11, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a12, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a13, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, 0x43090000
+; GFX908-NEXT: v_mov_b32_e32 v1, 0x430a0000
+; GFX908-NEXT: v_mov_b32_e32 v2, 0x430b0000
+; GFX908-NEXT: v_accvgpr_write_b32 a14, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a15, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a16, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, 0x430c0000
+; GFX908-NEXT: v_mov_b32_e32 v1, 0x430d0000
+; GFX908-NEXT: v_mov_b32_e32 v2, 0x430e0000
+; GFX908-NEXT: v_accvgpr_write_b32 a17, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a18, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a19, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, 0x430f0000
+; GFX908-NEXT: v_mov_b32_e32 v1, 0x43100000
+; GFX908-NEXT: v_mov_b32_e32 v2, 0x43110000
+; GFX908-NEXT: v_accvgpr_write_b32 a20, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a21, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a22, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, 0x43120000
+; GFX908-NEXT: v_mov_b32_e32 v1, 0x43130000
+; GFX908-NEXT: v_mov_b32_e32 v2, 0x43140000
+; GFX908-NEXT: v_accvgpr_write_b32 a23, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a24, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a25, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, 0x43150000
+; GFX908-NEXT: v_mov_b32_e32 v1, 0x43160000
+; GFX908-NEXT: v_mov_b32_e32 v2, 0x43170000
+; GFX908-NEXT: v_accvgpr_write_b32 a26, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a27, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a28, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, 0x43180000
+; GFX908-NEXT: v_mov_b32_e32 v1, 0x43190000
+; GFX908-NEXT: v_mov_b32_e32 v2, 0x431a0000
+; GFX908-NEXT: v_accvgpr_write_b32 a29, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a30, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a31, v2
; GFX908-NEXT: s_mov_b32 s0, 16
; GFX908-NEXT: v_mov_b32_e32 v0, 2.0
; GFX908-NEXT: v_mov_b32_e32 v1, 1.0
@@ -819,39 +821,39 @@ define amdgpu_kernel void @test_mfma_loop_unfoldable_seq(ptr addrspace(1) %arg)
;
; GFX90A-LABEL: test_mfma_loop_unfoldable_seq:
; GFX90A: ; %bb.0: ; %entry
-; GFX90A-NEXT: v_mov_b32_e32 v31, 0x431a0000
-; GFX90A-NEXT: v_mov_b32_e32 v30, 0x43190000
-; GFX90A-NEXT: v_mov_b32_e32 v29, 0x43180000
-; GFX90A-NEXT: v_mov_b32_e32 v28, 0x43170000
-; GFX90A-NEXT: v_mov_b32_e32 v27, 0x43160000
-; GFX90A-NEXT: v_mov_b32_e32 v26, 0x43150000
-; GFX90A-NEXT: v_mov_b32_e32 v25, 0x43140000
-; GFX90A-NEXT: v_mov_b32_e32 v24, 0x43130000
-; GFX90A-NEXT: v_mov_b32_e32 v23, 0x43120000
-; GFX90A-NEXT: v_mov_b32_e32 v22, 0x43110000
-; GFX90A-NEXT: v_mov_b32_e32 v21, 0x43100000
-; GFX90A-NEXT: v_mov_b32_e32 v20, 0x430f0000
-; GFX90A-NEXT: v_mov_b32_e32 v19, 0x430e0000
-; GFX90A-NEXT: v_mov_b32_e32 v18, 0x430d0000
-; GFX90A-NEXT: v_mov_b32_e32 v17, 0x430c0000
-; GFX90A-NEXT: v_mov_b32_e32 v16, 0x430b0000
-; GFX90A-NEXT: v_mov_b32_e32 v15, 0x430a0000
-; GFX90A-NEXT: v_mov_b32_e32 v14, 0x43090000
-; GFX90A-NEXT: v_mov_b32_e32 v13, 0x43080000
-; GFX90A-NEXT: v_mov_b32_e32 v12, 0x43070000
-; GFX90A-NEXT: v_mov_b32_e32 v11, 0x43060000
-; GFX90A-NEXT: v_mov_b32_e32 v10, 0x43050000
-; GFX90A-NEXT: v_mov_b32_e32 v9, 0x43040000
-; GFX90A-NEXT: v_mov_b32_e32 v8, 0x43030000
-; GFX90A-NEXT: v_mov_b32_e32 v7, 0x43020000
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0x43010000
-; GFX90A-NEXT: v_mov_b32_e32 v5, 0x43000000
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0x42fe0000
-; GFX90A-NEXT: v_mov_b32_e32 v3, 0x42fc0000
-; GFX90A-NEXT: v_mov_b32_e32 v2, 0x42fa0000
-; GFX90A-NEXT: v_mov_b32_e32 v1, 0x42f80000
-; GFX90A-NEXT: v_mov_b32_e32 v0, 0x42f60000
; GFX90A-NEXT: s_mov_b32 s0, 16
+; GFX90A-NEXT: v_mov_b32_e32 v0, 0x42f60000
+; GFX90A-NEXT: v_mov_b32_e32 v1, 0x42f80000
+; GFX90A-NEXT: v_mov_b32_e32 v2, 0x42fa0000
+; GFX90A-NEXT: v_mov_b32_e32 v3, 0x42fc0000
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0x42fe0000
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0x43000000
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0x43010000
+; GFX90A-NEXT: v_mov_b32_e32 v7, 0x43020000
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0x43030000
+; GFX90A-NEXT: v_mov_b32_e32 v9, 0x43040000
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0x43050000
+; GFX90A-NEXT: v_mov_b32_e32 v11, 0x43060000
+; GFX90A-NEXT: v_mov_b32_e32 v12, 0x43070000
+; GFX90A-NEXT: v_mov_b32_e32 v13, 0x43080000
+; GFX90A-NEXT: v_mov_b32_e32 v14, 0x43090000
+; GFX90A-NEXT: v_mov_b32_e32 v15, 0x430a0000
+; GFX90A-NEXT: v_mov_b32_e32 v16, 0x430b0000
+; GFX90A-NEXT: v_mov_b32_e32 v17, 0x430c0000
+; GFX90A-NEXT: v_mov_b32_e32 v18, 0x430d0000
+; GFX90A-NEXT: v_mov_b32_e32 v19, 0x430e0000
+; GFX90A-NEXT: v_mov_b32_e32 v20, 0x430f0000
+; GFX90A-NEXT: v_mov_b32_e32 v21, 0x43100000
+; GFX90A-NEXT: v_mov_b32_e32 v22, 0x43110000
+; GFX90A-NEXT: v_mov_b32_e32 v23, 0x43120000
+; GFX90A-NEXT: v_mov_b32_e32 v24, 0x43130000
+; GFX90A-NEXT: v_mov_b32_e32 v25, 0x43140000
+; GFX90A-NEXT: v_mov_b32_e32 v26, 0x43150000
+; GFX90A-NEXT: v_mov_b32_e32 v27, 0x43160000
+; GFX90A-NEXT: v_mov_b32_e32 v28, 0x43170000
+; GFX90A-NEXT: v_mov_b32_e32 v29, 0x43180000
+; GFX90A-NEXT: v_mov_b32_e32 v30, 0x43190000
+; GFX90A-NEXT: v_mov_b32_e32 v31, 0x431a0000
; GFX90A-NEXT: v_mov_b32_e32 v32, 2.0
; GFX90A-NEXT: v_mov_b32_e32 v33, 1.0
; GFX90A-NEXT: .LBB3_1: ; %for.cond.preheader
@@ -878,39 +880,39 @@ define amdgpu_kernel void @test_mfma_loop_unfoldable_seq(ptr addrspace(1) %arg)
;
; GFX942-LABEL: test_mfma_loop_unfoldable_seq:
; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: v_mov_b32_e32 v31, 0x431a0000
-; GFX942-NEXT: v_mov_b32_e32 v30, 0x43190000
-; GFX942-NEXT: v_mov_b32_e32 v29, 0x43180000
-; GFX942-NEXT: v_mov_b32_e32 v28, 0x43170000
-; GFX942-NEXT: v_mov_b32_e32 v27, 0x43160000
-; GFX942-NEXT: v_mov_b32_e32 v26, 0x43150000
-; GFX942-NEXT: v_mov_b32_e32 v25, 0x43140000
-; GFX942-NEXT: v_mov_b32_e32 v24, 0x43130000
-; GFX942-NEXT: v_mov_b32_e32 v23, 0x43120000
-; GFX942-NEXT: v_mov_b32_e32 v22, 0x43110000
-; GFX942-NEXT: v_mov_b32_e32 v21, 0x43100000
-; GFX942-NEXT: v_mov_b32_e32 v20, 0x430f0000
-; GFX942-NEXT: v_mov_b32_e32 v19, 0x430e0000
-; GFX942-NEXT: v_mov_b32_e32 v18, 0x430d0000
-; GFX942-NEXT: v_mov_b32_e32 v17, 0x430c0000
-; GFX942-NEXT: v_mov_b32_e32 v16, 0x430b0000
-; GFX942-NEXT: v_mov_b32_e32 v15, 0x430a0000
-; GFX942-NEXT: v_mov_b32_e32 v14, 0x43090000
-; GFX942-NEXT: v_mov_b32_e32 v13, 0x43080000
-; GFX942-NEXT: v_mov_b32_e32 v12, 0x43070000
-; GFX942-NEXT: v_mov_b32_e32 v11, 0x43060000
-; GFX942-NEXT: v_mov_b32_e32 v10, 0x43050000
-; GFX942-NEXT: v_mov_b32_e32 v9, 0x43040000
-; GFX942-NEXT: v_mov_b32_e32 v8, 0x43030000
-; GFX942-NEXT: v_mov_b32_e32 v7, 0x43020000
-; GFX942-NEXT: v_mov_b32_e32 v6, 0x43010000
-; GFX942-NEXT: v_mov_b32_e32 v5, 0x43000000
-; GFX942-NEXT: v_mov_b32_e32 v4, 0x42fe0000
-; GFX942-NEXT: v_mov_b32_e32 v3, 0x42fc0000
-; GFX942-NEXT: v_mov_b32_e32 v2, 0x42fa0000
-; GFX942-NEXT: v_mov_b32_e32 v1, 0x42f80000
-; GFX942-NEXT: v_mov_b32_e32 v0, 0x42f60000
; GFX942-NEXT: s_mov_b32 s0, 16
+; GFX942-NEXT: v_mov_b32_e32 v0, 0x42f60000
+; GFX942-NEXT: v_mov_b32_e32 v1, 0x42f80000
+; GFX942-NEXT: v_mov_b32_e32 v2, 0x42fa0000
+; GFX942-NEXT: v_mov_b32_e32 v3, 0x42fc0000
+; GFX942-NEXT: v_mov_b32_e32 v4, 0x42fe0000
+; GFX942-NEXT: v_mov_b32_e32 v5, 0x43000000
+; GFX942-NEXT: v_mov_b32_e32 v6, 0x43010000
+; GFX942-NEXT: v_mov_b32_e32 v7, 0x43020000
+; GFX942-NEXT: v_mov_b32_e32 v8, 0x43030000
+; GFX942-NEXT: v_mov_b32_e32 v9, 0x43040000
+; GFX942-NEXT: v_mov_b32_e32 v10, 0x43050000
+; GFX942-NEXT: v_mov_b32_e32 v11, 0x43060000
+; GFX942-NEXT: v_mov_b32_e32 v12, 0x43070000
+; GFX942-NEXT: v_mov_b32_e32 v13, 0x43080000
+; GFX942-NEXT: v_mov_b32_e32 v14, 0x43090000
+; GFX942-NEXT: v_mov_b32_e32 v15, 0x430a0000
+; GFX942-NEXT: v_mov_b32_e32 v16, 0x430b0000
+; GFX942-NEXT: v_mov_b32_e32 v17, 0x430c0000
+; GFX942-NEXT: v_mov_b32_e32 v18, 0x430d0000
+; GFX942-NEXT: v_mov_b32_e32 v19, 0x430e0000
+; GFX942-NEXT: v_mov_b32_e32 v20, 0x430f0000
+; GFX942-NEXT: v_mov_b32_e32 v21, 0x43100000
+; GFX942-NEXT: v_mov_b32_e32 v22, 0x43110000
+; GFX942-NEXT: v_mov_b32_e32 v23, 0x43120000
+; GFX942-NEXT: v_mov_b32_e32 v24, 0x43130000
+; GFX942-NEXT: v_mov_b32_e32 v25, 0x43140000
+; GFX942-NEXT: v_mov_b32_e32 v26, 0x43150000
+; GFX942-NEXT: v_mov_b32_e32 v27, 0x43160000
+; GFX942-NEXT: v_mov_b32_e32 v28, 0x43170000
+; GFX942-NEXT: v_mov_b32_e32 v29, 0x43180000
+; GFX942-NEXT: v_mov_b32_e32 v30, 0x43190000
+; GFX942-NEXT: v_mov_b32_e32 v31, 0x431a0000
; GFX942-NEXT: v_mov_b32_e32 v32, 2.0
; GFX942-NEXT: v_mov_b32_e32 v33, 1.0
; GFX942-NEXT: .LBB3_1: ; %for.cond.preheader
@@ -1481,39 +1483,39 @@ define amdgpu_kernel void @test_mfma_loop_mixed_init(ptr addrspace(1) %arg, floa
; GFX908: ; %bb.0: ; %entry
; GFX908-NEXT: s_load_dword s1, s[4:5], 0x2c
; GFX908-NEXT: v_accvgpr_write_b32 a0, v0
-; GFX908-NEXT: v_accvgpr_write_b32 a31, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a30, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a29, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a2, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a3, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a4, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v0, s1
-; GFX908-NEXT: v_accvgpr_write_b32 a28, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a27, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a26, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a25, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a24, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a23, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a22, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a21, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a20, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a19, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a18, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a17, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a16, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a15, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a14, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a13, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a12, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a11, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a10, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a9, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a8, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a7, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a6, 0
; GFX908-NEXT: v_accvgpr_write_b32 a5, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a4, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a3, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a2, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a6, 0
; GFX908-NEXT: v_accvgpr_write_b32 a1, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a7, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a8, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a9, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a10, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a11, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a12, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a13, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a14, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a15, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a16, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a17, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a18, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a19, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a20, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a21, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a22, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a23, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a24, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a25, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a26, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a27, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a28, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a29, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a30, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a31, 0
; GFX908-NEXT: s_mov_b32 s0, 16
; GFX908-NEXT: v_mov_b32_e32 v0, 2.0
; GFX908-NEXT: v_mov_b32_e32 v1, 1.0
@@ -1574,42 +1576,42 @@ define amdgpu_kernel void @test_mfma_loop_mixed_init(ptr addrspace(1) %arg, floa
; GFX90A-LABEL: test_mfma_loop_mixed_init:
; GFX90A: ; %bb.0: ; %entry
; GFX90A-NEXT: s_load_dword s1, s[4:5], 0x2c
-; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: s_mov_b32 s0, 16
+; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX90A-NEXT: v_mov_b32_e32 v32, 2.0
+; GFX90A-NEXT: v_mov_b32_e32 v3, 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s1
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
+; GFX90A-NEXT: v_mov_b32_e32 v7, 0
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-NEXT: v_mov_b32_e32 v9, 0
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
+; GFX90A-NEXT: v_mov_b32_e32 v11, 0
+; GFX90A-NEXT: v_mov_b32_e32 v12, 0
+; GFX90A-NEXT: v_mov_b32_e32 v13, 0
+; GFX90A-NEXT: v_mov_b32_e32 v14, 0
+; GFX90A-NEXT: v_mov_b32_e32 v15, 0
+; GFX90A-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-NEXT: v_mov_b32_e32 v17, 0
+; GFX90A-NEXT: v_mov_b32_e32 v18, 0
+; GFX90A-NEXT: v_mov_b32_e32 v19, 0
+; GFX90A-NEXT: v_mov_b32_e32 v20, 0
+; GFX90A-NEXT: v_mov_b32_e32 v21, 0
+; GFX90A-NEXT: v_mov_b32_e32 v22, 0
+; GFX90A-NEXT: v_mov_b32_e32 v23, 0
+; GFX90A-NEXT: v_mov_b32_e32 v24, 0
+; GFX90A-NEXT: v_mov_b32_e32 v25, 0
+; GFX90A-NEXT: v_mov_b32_e32 v26, 0
+; GFX90A-NEXT: v_mov_b32_e32 v27, 0
+; GFX90A-NEXT: v_mov_b32_e32 v28, 0
+; GFX90A-NEXT: v_mov_b32_e32 v29, 0
+; GFX90A-NEXT: v_mov_b32_e32 v30, 0
+; GFX90A-NEXT: v_mov_b32_e32 v31, 0
+; GFX90A-NEXT: v_mov_b32_e32 v32, 2.0
; GFX90A-NEXT: v_mov_b32_e32 v33, 1.0
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: v_mov_b32_e32 v4, v2
-; GFX90A-NEXT: v_mov_b32_e32 v5, v2
-; GFX90A-NEXT: v_mov_b32_e32 v6, v2
-; GFX90A-NEXT: v_mov_b32_e32 v7, v2
-; GFX90A-NEXT: v_mov_b32_e32 v8, v2
-; GFX90A-NEXT: v_mov_b32_e32 v9, v2
-; GFX90A-NEXT: v_mov_b32_e32 v10, v2
-; GFX90A-NEXT: v_mov_b32_e32 v11, v2
-; GFX90A-NEXT: v_mov_b32_e32 v12, v2
-; GFX90A-NEXT: v_mov_b32_e32 v13, v2
-; GFX90A-NEXT: v_mov_b32_e32 v14, v2
-; GFX90A-NEXT: v_mov_b32_e32 v15, v2
-; GFX90A-NEXT: v_mov_b32_e32 v16, v2
-; GFX90A-NEXT: v_mov_b32_e32 v17, v2
-; GFX90A-NEXT: v_mov_b32_e32 v18, v2
-; GFX90A-NEXT: v_mov_b32_e32 v19, v2
-; GFX90A-NEXT: v_mov_b32_e32 v20, v2
-; GFX90A-NEXT: v_mov_b32_e32 v21, v2
-; GFX90A-NEXT: v_mov_b32_e32 v22, v2
-; GFX90A-NEXT: v_mov_b32_e32 v23, v2
-; GFX90A-NEXT: v_mov_b32_e32 v24, v2
-; GFX90A-NEXT: v_mov_b32_e32 v25, v2
-; GFX90A-NEXT: v_mov_b32_e32 v26, v2
-; GFX90A-NEXT: v_mov_b32_e32 v27, v2
-; GFX90A-NEXT: v_mov_b32_e32 v28, v2
-; GFX90A-NEXT: v_mov_b32_e32 v29, v2
-; GFX90A-NEXT: v_mov_b32_e32 v30, v2
-; GFX90A-NEXT: v_mov_b32_e32 v31, v2
; GFX90A-NEXT: .LBB6_1: ; %for.cond.preheader
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_nop 1
@@ -1635,42 +1637,42 @@ define amdgpu_kernel void @test_mfma_loop_mixed_init(ptr addrspace(1) %arg, floa
; GFX942-LABEL: test_mfma_loop_mixed_init:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_load_dword s1, s[4:5], 0x2c
-; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: s_mov_b32 s0, 16
+; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-NEXT: v_mov_b32_e32 v32, 2.0
+; GFX942-NEXT: v_mov_b32_e32 v3, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, s1
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-NEXT: v_mov_b32_e32 v5, 0
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v7, 0
+; GFX942-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-NEXT: v_mov_b32_e32 v9, 0
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
+; GFX942-NEXT: v_mov_b32_e32 v11, 0
+; GFX942-NEXT: v_mov_b32_e32 v12, 0
+; GFX942-NEXT: v_mov_b32_e32 v13, 0
+; GFX942-NEXT: v_mov_b32_e32 v14, 0
+; GFX942-NEXT: v_mov_b32_e32 v15, 0
+; GFX942-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-NEXT: v_mov_b32_e32 v17, 0
+; GFX942-NEXT: v_mov_b32_e32 v18, 0
+; GFX942-NEXT: v_mov_b32_e32 v19, 0
+; GFX942-NEXT: v_mov_b32_e32 v20, 0
+; GFX942-NEXT: v_mov_b32_e32 v21, 0
+; GFX942-NEXT: v_mov_b32_e32 v22, 0
+; GFX942-NEXT: v_mov_b32_e32 v23, 0
+; GFX942-NEXT: v_mov_b32_e32 v24, 0
+; GFX942-NEXT: v_mov_b32_e32 v25, 0
+; GFX942-NEXT: v_mov_b32_e32 v26, 0
+; GFX942-NEXT: v_mov_b32_e32 v27, 0
+; GFX942-NEXT: v_mov_b32_e32 v28, 0
+; GFX942-NEXT: v_mov_b32_e32 v29, 0
+; GFX942-NEXT: v_mov_b32_e32 v30, 0
+; GFX942-NEXT: v_mov_b32_e32 v31, 0
+; GFX942-NEXT: v_mov_b32_e32 v32, 2.0
; GFX942-NEXT: v_mov_b32_e32 v33, 1.0
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_mov_b32_e32 v4, v2
-; GFX942-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-NEXT: v_mov_b32_e32 v6, v2
-; GFX942-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-NEXT: v_mov_b32_e32 v8, v2
-; GFX942-NEXT: v_mov_b32_e32 v9, v2
-; GFX942-NEXT: v_mov_b32_e32 v10, v2
-; GFX942-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-NEXT: v_mov_b32_e32 v12, v2
-; GFX942-NEXT: v_mov_b32_e32 v13, v2
-; GFX942-NEXT: v_mov_b32_e32 v14, v2
-; GFX942-NEXT: v_mov_b32_e32 v15, v2
-; GFX942-NEXT: v_mov_b32_e32 v16, v2
-; GFX942-NEXT: v_mov_b32_e32 v17, v2
-; GFX942-NEXT: v_mov_b32_e32 v18, v2
-; GFX942-NEXT: v_mov_b32_e32 v19, v2
-; GFX942-NEXT: v_mov_b32_e32 v20, v2
-; GFX942-NEXT: v_mov_b32_e32 v21, v2
-; GFX942-NEXT: v_mov_b32_e32 v22, v2
-; GFX942-NEXT: v_mov_b32_e32 v23, v2
-; GFX942-NEXT: v_mov_b32_e32 v24, v2
-; GFX942-NEXT: v_mov_b32_e32 v25, v2
-; GFX942-NEXT: v_mov_b32_e32 v26, v2
-; GFX942-NEXT: v_mov_b32_e32 v27, v2
-; GFX942-NEXT: v_mov_b32_e32 v28, v2
-; GFX942-NEXT: v_mov_b32_e32 v29, v2
-; GFX942-NEXT: v_mov_b32_e32 v30, v2
-; GFX942-NEXT: v_mov_b32_e32 v31, v2
; GFX942-NEXT: .LBB6_1: ; %for.cond.preheader
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_nop 1
@@ -2191,57 +2193,120 @@ exit:
define amdgpu_kernel void @test_mfma_nested_loop_zeroinit(ptr addrspace(1) %arg) #0 {
; GFX908-LABEL: test_mfma_nested_loop_zeroinit:
; GFX908: ; %bb.0: ; %entry
-; GFX908-NEXT: v_accvgpr_write_b32 a0, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a1, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a2, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a3, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a4, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a5, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a6, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a7, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a8, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a9, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a10, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a11, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a12, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a13, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a14, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a15, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a16, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a17, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a18, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a19, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a20, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a21, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a22, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a23, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a24, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a25, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a26, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a27, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a28, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a29, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a30, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a31, 0
-; GFX908-NEXT: s_mov_b32 s0, 0
+; GFX908-NEXT: s_mov_b32 s36, 0
+; GFX908-NEXT: s_mov_b32 s37, s36
+; GFX908-NEXT: s_mov_b32 s38, s36
+; GFX908-NEXT: s_mov_b32 s39, s36
+; GFX908-NEXT: s_mov_b32 s40, s36
+; GFX908-NEXT: s_mov_b32 s41, s36
+; GFX908-NEXT: s_mov_b32 s42, s36
+; GFX908-NEXT: s_mov_b32 s43, s36
+; GFX908-NEXT: s_mov_b32 s44, s36
+; GFX908-NEXT: s_mov_b32 s45, s36
+; GFX908-NEXT: s_mov_b32 s46, s36
+; GFX908-NEXT: s_mov_b32 s47, s36
+; GFX908-NEXT: s_mov_b32 s48, s36
+; GFX908-NEXT: s_mov_b32 s49, s36
+; GFX908-NEXT: s_mov_b32 s50, s36
+; GFX908-NEXT: s_mov_b32 s51, s36
+; GFX908-NEXT: s_mov_b32 s52, s36
+; GFX908-NEXT: s_mov_b32 s53, s36
+; GFX908-NEXT: s_mov_b32 s54, s36
+; GFX908-NEXT: s_mov_b32 s55, s36
+; GFX908-NEXT: s_mov_b32 s56, s36
+; GFX908-NEXT: s_mov_b32 s57, s36
+; GFX908-NEXT: s_mov_b32 s58, s36
+; GFX908-NEXT: s_mov_b32 s59, s36
+; GFX908-NEXT: s_mov_b32 s60, s36
+; GFX908-NEXT: s_mov_b32 s61, s36
+; GFX908-NEXT: s_mov_b32 s62, s36
+; GFX908-NEXT: s_mov_b32 s63, s36
+; GFX908-NEXT: s_mov_b32 s64, s36
+; GFX908-NEXT: s_mov_b32 s65, s36
+; GFX908-NEXT: s_mov_b32 s66, s36
+; GFX908-NEXT: s_mov_b32 s67, s36
+; GFX908-NEXT: v_mov_b32_e32 v0, s36
+; GFX908-NEXT: v_mov_b32_e32 v1, s37
+; GFX908-NEXT: v_mov_b32_e32 v2, s38
+; GFX908-NEXT: v_mov_b32_e32 v3, s39
+; GFX908-NEXT: v_mov_b32_e32 v4, s40
+; GFX908-NEXT: v_mov_b32_e32 v5, s41
+; GFX908-NEXT: v_mov_b32_e32 v6, s42
+; GFX908-NEXT: v_mov_b32_e32 v7, s43
+; GFX908-NEXT: v_mov_b32_e32 v8, s44
+; GFX908-NEXT: v_mov_b32_e32 v9, s45
+; GFX908-NEXT: v_mov_b32_e32 v10, s46
+; GFX908-NEXT: v_mov_b32_e32 v11, s47
+; GFX908-NEXT: v_mov_b32_e32 v12, s48
+; GFX908-NEXT: v_mov_b32_e32 v13, s49
+; GFX908-NEXT: v_mov_b32_e32 v14, s50
+; GFX908-NEXT: v_mov_b32_e32 v15, s51
+; GFX908-NEXT: v_mov_b32_e32 v16, s52
+; GFX908-NEXT: v_mov_b32_e32 v17, s53
+; GFX908-NEXT: v_mov_b32_e32 v18, s54
+; GFX908-NEXT: v_mov_b32_e32 v19, s55
+; GFX908-NEXT: v_mov_b32_e32 v20, s56
+; GFX908-NEXT: v_mov_b32_e32 v21, s57
+; GFX908-NEXT: v_mov_b32_e32 v22, s58
+; GFX908-NEXT: v_mov_b32_e32 v23, s59
+; GFX908-NEXT: v_mov_b32_e32 v24, s60
+; GFX908-NEXT: v_mov_b32_e32 v25, s61
+; GFX908-NEXT: v_mov_b32_e32 v26, s62
+; GFX908-NEXT: v_mov_b32_e32 v27, s63
+; GFX908-NEXT: v_mov_b32_e32 v28, s64
+; GFX908-NEXT: v_mov_b32_e32 v29, s65
+; GFX908-NEXT: v_mov_b32_e32 v30, s66
+; GFX908-NEXT: v_mov_b32_e32 v31, s67
+; GFX908-NEXT: v_accvgpr_write_b32 a0, v0
+; GFX908-NEXT: v_accvgpr_write_b32 a1, v1
+; GFX908-NEXT: v_accvgpr_write_b32 a2, v2
+; GFX908-NEXT: v_accvgpr_write_b32 a3, v3
+; GFX908-NEXT: v_accvgpr_write_b32 a4, v4
+; GFX908-NEXT: v_accvgpr_write_b32 a5, v5
+; GFX908-NEXT: v_accvgpr_write_b32 a6, v6
+; GFX908-NEXT: v_accvgpr_write_b32 a7, v7
+; GFX908-NEXT: v_accvgpr_write_b32 a8, v8
+; GFX908-NEXT: v_accvgpr_write_b32 a9, v9
+; GFX908-NEXT: v_accvgpr_write_b32 a10, v10
+; GFX908-NEXT: v_accvgpr_write_b32 a11, v11
+; GFX908-NEXT: v_accvgpr_write_b32 a12, v12
+; GFX908-NEXT: v_accvgpr_write_b32 a13, v13
+; GFX908-NEXT: v_accvgpr_write_b32 a14, v14
+; GFX908-NEXT: v_accvgpr_write_b32 a15, v15
+; GFX908-NEXT: v_accvgpr_write_b32 a16, v16
+; GFX908-NEXT: v_accvgpr_write_b32 a17, v17
+; GFX908-NEXT: v_accvgpr_write_b32 a18, v18
+; GFX908-NEXT: v_accvgpr_write_b32 a19, v19
+; GFX908-NEXT: v_accvgpr_write_b32 a20, v20
+; GFX908-NEXT: v_accvgpr_write_b32 a21, v21
+; GFX908-NEXT: v_accvgpr_write_b32 a22, v22
+; GFX908-NEXT: v_accvgpr_write_b32 a23, v23
+; GFX908-NEXT: v_accvgpr_write_b32 a24, v24
+; GFX908-NEXT: v_accvgpr_write_b32 a25, v25
+; GFX908-NEXT: v_accvgpr_write_b32 a26, v26
+; GFX908-NEXT: v_accvgpr_write_b32 a27, v27
+; GFX908-NEXT: v_accvgpr_write_b32 a28, v28
+; GFX908-NEXT: v_accvgpr_write_b32 a29, v29
+; GFX908-NEXT: v_accvgpr_write_b32 a30, v30
+; GFX908-NEXT: v_accvgpr_write_b32 a31, v31
; GFX908-NEXT: v_mov_b32_e32 v0, 2.0
; GFX908-NEXT: v_mov_b32_e32 v1, 1.0
; GFX908-NEXT: .LBB9_1: ; %for.cond.preheader
; GFX908-NEXT: ; =>This Loop Header: Depth=1
; GFX908-NEXT: ; Child Loop BB9_2 Depth 2
-; GFX908-NEXT: s_mov_b32 s1, 16
+; GFX908-NEXT: s_mov_b32 s0, 16
; GFX908-NEXT: .LBB9_2: ; %inner.for.cond.preheader
; GFX908-NEXT: ; Parent Loop BB9_1 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: s_nop 0
; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v0, a[0:31]
-; GFX908-NEXT: s_add_i32 s1, s1, -1
-; GFX908-NEXT: s_cmp_lg_u32 s1, 0
+; GFX908-NEXT: s_add_i32 s0, s0, -1
+; GFX908-NEXT: s_cmp_lg_u32 s0, 0
; GFX908-NEXT: s_cbranch_scc1 .LBB9_2
; GFX908-NEXT: ; %bb.3: ; %inner.exit
; GFX908-NEXT: ; in Loop: Header=BB9_1 Depth=1
-; GFX908-NEXT: s_add_i32 s0, s0, 1
-; GFX908-NEXT: s_cmp_lg_u32 s0, 16
+; GFX908-NEXT: s_add_i32 s36, s36, 1
+; GFX908-NEXT: s_cmp_lg_u32 s36, 16
; GFX908-NEXT: s_cbranch_scc1 .LBB9_1
; GFX908-NEXT: ; %bb.4: ; %exit
; GFX908-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
@@ -2292,57 +2357,72 @@ define amdgpu_kernel void @test_mfma_nested_loop_zeroinit(ptr addrspace(1) %arg)
;
; GFX90A-LABEL: test_mfma_nested_loop_zeroinit:
; GFX90A: ; %bb.0: ; %entry
-; GFX90A-NEXT: v_mov_b32_e32 v0, 0
-; GFX90A-NEXT: s_mov_b32 s0, 0
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: v_mov_b32_e32 v4, v0
-; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_mov_b32_e32 v6, v0
-; GFX90A-NEXT: v_mov_b32_e32 v7, v0
-; GFX90A-NEXT: v_mov_b32_e32 v8, v0
-; GFX90A-NEXT: v_mov_b32_e32 v9, v0
-; GFX90A-NEXT: v_mov_b32_e32 v10, v0
-; GFX90A-NEXT: v_mov_b32_e32 v11, v0
-; GFX90A-NEXT: v_mov_b32_e32 v12, v0
-; GFX90A-NEXT: v_mov_b32_e32 v13, v0
-; GFX90A-NEXT: v_mov_b32_e32 v14, v0
-; GFX90A-NEXT: v_mov_b32_e32 v15, v0
-; GFX90A-NEXT: v_mov_b32_e32 v16, v0
-; GFX90A-NEXT: v_mov_b32_e32 v17, v0
-; GFX90A-NEXT: v_mov_b32_e32 v18, v0
-; GFX90A-NEXT: v_mov_b32_e32 v19, v0
-; GFX90A-NEXT: v_mov_b32_e32 v20, v0
-; GFX90A-NEXT: v_mov_b32_e32 v21, v0
-; GFX90A-NEXT: v_mov_b32_e32 v22, v0
-; GFX90A-NEXT: v_mov_b32_e32 v23, v0
-; GFX90A-NEXT: v_mov_b32_e32 v24, v0
-; GFX90A-NEXT: v_mov_b32_e32 v25, v0
-; GFX90A-NEXT: v_mov_b32_e32 v26, v0
-; GFX90A-NEXT: v_mov_b32_e32 v27, v0
-; GFX90A-NEXT: v_mov_b32_e32 v28, v0
-; GFX90A-NEXT: v_mov_b32_e32 v29, v0
-; GFX90A-NEXT: v_mov_b32_e32 v30, v0
-; GFX90A-NEXT: v_mov_b32_e32 v31, v0
+; GFX90A-NEXT: s_mov_b32 s36, 0
+; GFX90A-NEXT: s_mov_b32 s37, s36
+; GFX90A-NEXT: s_mov_b32 s38, s36
+; GFX90A-NEXT: s_mov_b32 s39, s36
+; GFX90A-NEXT: s_mov_b32 s40, s36
+; GFX90A-NEXT: s_mov_b32 s41, s36
+; GFX90A-NEXT: s_mov_b32 s42, s36
+; GFX90A-NEXT: s_mov_b32 s43, s36
+; GFX90A-NEXT: s_mov_b32 s44, s36
+; GFX90A-NEXT: s_mov_b32 s45, s36
+; GFX90A-NEXT: s_mov_b32 s46, s36
+; GFX90A-NEXT: s_mov_b32 s47, s36
+; GFX90A-NEXT: s_mov_b32 s48, s36
+; GFX90A-NEXT: s_mov_b32 s49, s36
+; GFX90A-NEXT: s_mov_b32 s50, s36
+; GFX90A-NEXT: s_mov_b32 s51, s36
+; GFX90A-NEXT: s_mov_b32 s52, s36
+; GFX90A-NEXT: s_mov_b32 s53, s36
+; GFX90A-NEXT: s_mov_b32 s54, s36
+; GFX90A-NEXT: s_mov_b32 s55, s36
+; GFX90A-NEXT: s_mov_b32 s56, s36
+; GFX90A-NEXT: s_mov_b32 s57, s36
+; GFX90A-NEXT: s_mov_b32 s58, s36
+; GFX90A-NEXT: s_mov_b32 s59, s36
+; GFX90A-NEXT: s_mov_b32 s60, s36
+; GFX90A-NEXT: s_mov_b32 s61, s36
+; GFX90A-NEXT: s_mov_b32 s62, s36
+; GFX90A-NEXT: s_mov_b32 s63, s36
+; GFX90A-NEXT: s_mov_b32 s64, s36
+; GFX90A-NEXT: s_mov_b32 s65, s36
+; GFX90A-NEXT: s_mov_b32 s66, s36
+; GFX90A-NEXT: s_mov_b32 s67, s36
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[36:37], s[36:37] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[38:39], s[38:39] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[40:41], s[40:41] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[42:43], s[42:43] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[8:9], s[44:45], s[44:45] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[10:11], s[46:47], s[46:47] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[12:13], s[48:49], s[48:49] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[14:15], s[50:51], s[50:51] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[16:17], s[52:53], s[52:53] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[18:19], s[54:55], s[54:55] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[20:21], s[56:57], s[56:57] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[22:23], s[58:59], s[58:59] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[24:25], s[60:61], s[60:61] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[26:27], s[62:63], s[62:63] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[28:29], s[64:65], s[64:65] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[30:31], s[66:67], s[66:67] op_sel:[0,1]
; GFX90A-NEXT: v_mov_b32_e32 v32, 2.0
; GFX90A-NEXT: v_mov_b32_e32 v33, 1.0
; GFX90A-NEXT: .LBB9_1: ; %for.cond.preheader
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
; GFX90A-NEXT: ; Child Loop BB9_2 Depth 2
-; GFX90A-NEXT: s_mov_b32 s1, 16
+; GFX90A-NEXT: s_mov_b32 s0, 16
; GFX90A-NEXT: .LBB9_2: ; %inner.for.cond.preheader
; GFX90A-NEXT: ; Parent Loop BB9_1 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
; GFX90A-NEXT: s_nop 0
; GFX90A-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v33, v32, v[0:31]
-; GFX90A-NEXT: s_add_i32 s1, s1, -1
-; GFX90A-NEXT: s_cmp_lg_u32 s1, 0
+; GFX90A-NEXT: s_add_i32 s0, s0, -1
+; GFX90A-NEXT: s_cmp_lg_u32 s0, 0
; GFX90A-NEXT: s_cbranch_scc1 .LBB9_2
; GFX90A-NEXT: ; %bb.3: ; %inner.exit
; GFX90A-NEXT: ; in Loop: Header=BB9_1 Depth=1
-; GFX90A-NEXT: s_add_i32 s0, s0, 1
-; GFX90A-NEXT: s_cmp_lg_u32 s0, 16
+; GFX90A-NEXT: s_add_i32 s36, s36, 1
+; GFX90A-NEXT: s_cmp_lg_u32 s36, 16
; GFX90A-NEXT: s_cbranch_scc1 .LBB9_1
; GFX90A-NEXT: ; %bb.4: ; %exit
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
@@ -2361,57 +2441,72 @@ define amdgpu_kernel void @test_mfma_nested_loop_zeroinit(ptr addrspace(1) %arg)
;
; GFX942-LABEL: test_mfma_nested_loop_zeroinit:
; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: s_mov_b32 s0, 0
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: v_mov_b32_e32 v4, v0
-; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_mov_b32_e32 v6, v0
-; GFX942-NEXT: v_mov_b32_e32 v7, v0
-; GFX942-NEXT: v_mov_b32_e32 v8, v0
-; GFX942-NEXT: v_mov_b32_e32 v9, v0
-; GFX942-NEXT: v_mov_b32_e32 v10, v0
-; GFX942-NEXT: v_mov_b32_e32 v11, v0
-; GFX942-NEXT: v_mov_b32_e32 v12, v0
-; GFX942-NEXT: v_mov_b32_e32 v13, v0
-; GFX942-NEXT: v_mov_b32_e32 v14, v0
-; GFX942-NEXT: v_mov_b32_e32 v15, v0
-; GFX942-NEXT: v_mov_b32_e32 v16, v0
-; GFX942-NEXT: v_mov_b32_e32 v17, v0
-; GFX942-NEXT: v_mov_b32_e32 v18, v0
-; GFX942-NEXT: v_mov_b32_e32 v19, v0
-; GFX942-NEXT: v_mov_b32_e32 v20, v0
-; GFX942-NEXT: v_mov_b32_e32 v21, v0
-; GFX942-NEXT: v_mov_b32_e32 v22, v0
-; GFX942-NEXT: v_mov_b32_e32 v23, v0
-; GFX942-NEXT: v_mov_b32_e32 v24, v0
-; GFX942-NEXT: v_mov_b32_e32 v25, v0
-; GFX942-NEXT: v_mov_b32_e32 v26, v0
-; GFX942-NEXT: v_mov_b32_e32 v27, v0
-; GFX942-NEXT: v_mov_b32_e32 v28, v0
-; GFX942-NEXT: v_mov_b32_e32 v29, v0
-; GFX942-NEXT: v_mov_b32_e32 v30, v0
-; GFX942-NEXT: v_mov_b32_e32 v31, v0
+; GFX942-NEXT: s_mov_b32 s36, 0
+; GFX942-NEXT: s_mov_b32 s37, s36
+; GFX942-NEXT: s_mov_b32 s38, s36
+; GFX942-NEXT: s_mov_b32 s39, s36
+; GFX942-NEXT: s_mov_b32 s40, s36
+; GFX942-NEXT: s_mov_b32 s41, s36
+; GFX942-NEXT: s_mov_b32 s42, s36
+; GFX942-NEXT: s_mov_b32 s43, s36
+; GFX942-NEXT: s_mov_b32 s44, s36
+; GFX942-NEXT: s_mov_b32 s45, s36
+; GFX942-NEXT: s_mov_b32 s46, s36
+; GFX942-NEXT: s_mov_b32 s47, s36
+; GFX942-NEXT: s_mov_b32 s48, s36
+; GFX942-NEXT: s_mov_b32 s49, s36
+; GFX942-NEXT: s_mov_b32 s50, s36
+; GFX942-NEXT: s_mov_b32 s51, s36
+; GFX942-NEXT: s_mov_b32 s52, s36
+; GFX942-NEXT: s_mov_b32 s53, s36
+; GFX942-NEXT: s_mov_b32 s54, s36
+; GFX942-NEXT: s_mov_b32 s55, s36
+; GFX942-NEXT: s_mov_b32 s56, s36
+; GFX942-NEXT: s_mov_b32 s57, s36
+; GFX942-NEXT: s_mov_b32 s58, s36
+; GFX942-NEXT: s_mov_b32 s59, s36
+; GFX942-NEXT: s_mov_b32 s60, s36
+; GFX942-NEXT: s_mov_b32 s61, s36
+; GFX942-NEXT: s_mov_b32 s62, s36
+; GFX942-NEXT: s_mov_b32 s63, s36
+; GFX942-NEXT: s_mov_b32 s64, s36
+; GFX942-NEXT: s_mov_b32 s65, s36
+; GFX942-NEXT: s_mov_b32 s66, s36
+; GFX942-NEXT: s_mov_b32 s67, s36
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[36:37]
+; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[38:39]
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], s[40:41]
+; GFX942-NEXT: v_mov_b64_e32 v[6:7], s[42:43]
+; GFX942-NEXT: v_mov_b64_e32 v[8:9], s[44:45]
+; GFX942-NEXT: v_mov_b64_e32 v[10:11], s[46:47]
+; GFX942-NEXT: v_mov_b64_e32 v[12:13], s[48:49]
+; GFX942-NEXT: v_mov_b64_e32 v[14:15], s[50:51]
+; GFX942-NEXT: v_mov_b64_e32 v[16:17], s[52:53]
+; GFX942-NEXT: v_mov_b64_e32 v[18:19], s[54:55]
+; GFX942-NEXT: v_mov_b64_e32 v[20:21], s[56:57]
+; GFX942-NEXT: v_mov_b64_e32 v[22:23], s[58:59]
+; GFX942-NEXT: v_mov_b64_e32 v[24:25], s[60:61]
+; GFX942-NEXT: v_mov_b64_e32 v[26:27], s[62:63]
+; GFX942-NEXT: v_mov_b64_e32 v[28:29], s[64:65]
+; GFX942-NEXT: v_mov_b64_e32 v[30:31], s[66:67]
; GFX942-NEXT: v_mov_b32_e32 v32, 2.0
; GFX942-NEXT: v_mov_b32_e32 v33, 1.0
; GFX942-NEXT: .LBB9_1: ; %for.cond.preheader
; GFX942-NEXT: ; =>This Loop Header: Depth=1
; GFX942-NEXT: ; Child Loop BB9_2 Depth 2
-; GFX942-NEXT: s_mov_b32 s1, 16
+; GFX942-NEXT: s_mov_b32 s0, 16
; GFX942-NEXT: .LBB9_2: ; %inner.for.cond.preheader
; GFX942-NEXT: ; Parent Loop BB9_1 Depth=1
; GFX942-NEXT: ; => This Inner Loop Header: Depth=2
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v33, v32, v[0:31]
-; GFX942-NEXT: s_add_i32 s1, s1, -1
-; GFX942-NEXT: s_cmp_lg_u32 s1, 0
+; GFX942-NEXT: s_add_i32 s0, s0, -1
+; GFX942-NEXT: s_cmp_lg_u32 s0, 0
; GFX942-NEXT: s_cbranch_scc1 .LBB9_2
; GFX942-NEXT: ; %bb.3: ; %inner.exit
; GFX942-NEXT: ; in Loop: Header=BB9_1 Depth=1
-; GFX942-NEXT: s_add_i32 s0, s0, 1
-; GFX942-NEXT: s_cmp_lg_u32 s0, 16
+; GFX942-NEXT: s_add_i32 s36, s36, 1
+; GFX942-NEXT: s_cmp_lg_u32 s36, 16
; GFX942-NEXT: s_cbranch_scc1 .LBB9_1
; GFX942-NEXT: ; %bb.4: ; %exit
; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
@@ -2458,38 +2553,38 @@ define <32 x float> @test_mfma_loop_zeroinit_ret_use() #0 {
; GFX908-LABEL: test_mfma_loop_zeroinit_ret_use:
; GFX908: ; %bb.0: ; %entry
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_accvgpr_write_b32 a31, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a30, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a29, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a28, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a27, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a26, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a25, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a24, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a23, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a22, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a21, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a20, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a19, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a18, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a17, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a16, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a15, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a14, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a13, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a12, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a11, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a10, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a9, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a8, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a7, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a6, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a5, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a4, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a3, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a2, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a1, 0
; GFX908-NEXT: v_accvgpr_write_b32 a0, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a1, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a2, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a3, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a4, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a5, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a6, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a7, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a8, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a9, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a10, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a11, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a12, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a13, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a14, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a15, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a16, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a17, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a18, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a19, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a20, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a21, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a22, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a23, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a24, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a25, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a26, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a27, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a28, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a29, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a30, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a31, 0
; GFX908-NEXT: s_mov_b32 s4, 16
; GFX908-NEXT: v_mov_b32_e32 v0, 2.0
; GFX908-NEXT: v_mov_b32_e32 v1, 1.0
@@ -2539,41 +2634,41 @@ define <32 x float> @test_mfma_loop_zeroinit_ret_use() #0 {
; GFX90A-LABEL: test_mfma_loop_zeroinit_ret_use:
; GFX90A: ; %bb.0: ; %entry
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_mov_b32 s4, 16
+; GFX90A-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-NEXT: v_mov_b32_e32 v1, 0
+; GFX90A-NEXT: v_mov_b32_e32 v2, 0
+; GFX90A-NEXT: v_mov_b32_e32 v3, 0
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
+; GFX90A-NEXT: v_mov_b32_e32 v7, 0
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-NEXT: v_mov_b32_e32 v9, 0
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
+; GFX90A-NEXT: v_mov_b32_e32 v11, 0
+; GFX90A-NEXT: v_mov_b32_e32 v12, 0
+; GFX90A-NEXT: v_mov_b32_e32 v13, 0
+; GFX90A-NEXT: v_mov_b32_e32 v14, 0
+; GFX90A-NEXT: v_mov_b32_e32 v15, 0
+; GFX90A-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-NEXT: v_mov_b32_e32 v17, 0
+; GFX90A-NEXT: v_mov_b32_e32 v18, 0
+; GFX90A-NEXT: v_mov_b32_e32 v19, 0
+; GFX90A-NEXT: v_mov_b32_e32 v20, 0
+; GFX90A-NEXT: v_mov_b32_e32 v21, 0
+; GFX90A-NEXT: v_mov_b32_e32 v22, 0
+; GFX90A-NEXT: v_mov_b32_e32 v23, 0
+; GFX90A-NEXT: v_mov_b32_e32 v24, 0
+; GFX90A-NEXT: v_mov_b32_e32 v25, 0
+; GFX90A-NEXT: v_mov_b32_e32 v26, 0
+; GFX90A-NEXT: v_mov_b32_e32 v27, 0
+; GFX90A-NEXT: v_mov_b32_e32 v28, 0
+; GFX90A-NEXT: v_mov_b32_e32 v29, 0
+; GFX90A-NEXT: v_mov_b32_e32 v30, 0
+; GFX90A-NEXT: v_mov_b32_e32 v31, 0
; GFX90A-NEXT: v_mov_b32_e32 v32, 2.0
; GFX90A-NEXT: v_mov_b32_e32 v33, 1.0
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: v_mov_b32_e32 v4, v0
-; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_mov_b32_e32 v6, v0
-; GFX90A-NEXT: v_mov_b32_e32 v7, v0
-; GFX90A-NEXT: v_mov_b32_e32 v8, v0
-; GFX90A-NEXT: v_mov_b32_e32 v9, v0
-; GFX90A-NEXT: v_mov_b32_e32 v10, v0
-; GFX90A-NEXT: v_mov_b32_e32 v11, v0
-; GFX90A-NEXT: v_mov_b32_e32 v12, v0
-; GFX90A-NEXT: v_mov_b32_e32 v13, v0
-; GFX90A-NEXT: v_mov_b32_e32 v14, v0
-; GFX90A-NEXT: v_mov_b32_e32 v15, v0
-; GFX90A-NEXT: v_mov_b32_e32 v16, v0
-; GFX90A-NEXT: v_mov_b32_e32 v17, v0
-; GFX90A-NEXT: v_mov_b32_e32 v18, v0
-; GFX90A-NEXT: v_mov_b32_e32 v19, v0
-; GFX90A-NEXT: v_mov_b32_e32 v20, v0
-; GFX90A-NEXT: v_mov_b32_e32 v21, v0
-; GFX90A-NEXT: v_mov_b32_e32 v22, v0
-; GFX90A-NEXT: v_mov_b32_e32 v23, v0
-; GFX90A-NEXT: v_mov_b32_e32 v24, v0
-; GFX90A-NEXT: v_mov_b32_e32 v25, v0
-; GFX90A-NEXT: v_mov_b32_e32 v26, v0
-; GFX90A-NEXT: v_mov_b32_e32 v27, v0
-; GFX90A-NEXT: v_mov_b32_e32 v28, v0
-; GFX90A-NEXT: v_mov_b32_e32 v29, v0
-; GFX90A-NEXT: v_mov_b32_e32 v30, v0
-; GFX90A-NEXT: v_mov_b32_e32 v31, v0
; GFX90A-NEXT: .LBB10_1: ; %for.cond.preheader
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_nop 1
@@ -2587,41 +2682,41 @@ define <32 x float> @test_mfma_loop_zeroinit_ret_use() #0 {
; GFX942-LABEL: test_mfma_loop_zeroinit_ret_use:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_mov_b32 s0, 16
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: v_mov_b32_e32 v1, 0
+; GFX942-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-NEXT: v_mov_b32_e32 v3, 0
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-NEXT: v_mov_b32_e32 v5, 0
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v7, 0
+; GFX942-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-NEXT: v_mov_b32_e32 v9, 0
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
+; GFX942-NEXT: v_mov_b32_e32 v11, 0
+; GFX942-NEXT: v_mov_b32_e32 v12, 0
+; GFX942-NEXT: v_mov_b32_e32 v13, 0
+; GFX942-NEXT: v_mov_b32_e32 v14, 0
+; GFX942-NEXT: v_mov_b32_e32 v15, 0
+; GFX942-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-NEXT: v_mov_b32_e32 v17, 0
+; GFX942-NEXT: v_mov_b32_e32 v18, 0
+; GFX942-NEXT: v_mov_b32_e32 v19, 0
+; GFX942-NEXT: v_mov_b32_e32 v20, 0
+; GFX942-NEXT: v_mov_b32_e32 v21, 0
+; GFX942-NEXT: v_mov_b32_e32 v22, 0
+; GFX942-NEXT: v_mov_b32_e32 v23, 0
+; GFX942-NEXT: v_mov_b32_e32 v24, 0
+; GFX942-NEXT: v_mov_b32_e32 v25, 0
+; GFX942-NEXT: v_mov_b32_e32 v26, 0
+; GFX942-NEXT: v_mov_b32_e32 v27, 0
+; GFX942-NEXT: v_mov_b32_e32 v28, 0
+; GFX942-NEXT: v_mov_b32_e32 v29, 0
+; GFX942-NEXT: v_mov_b32_e32 v30, 0
+; GFX942-NEXT: v_mov_b32_e32 v31, 0
; GFX942-NEXT: v_mov_b32_e32 v32, 2.0
; GFX942-NEXT: v_mov_b32_e32 v33, 1.0
-; GFX942-NEXT: v_mov_b32_e32 v1, v0
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: v_mov_b32_e32 v4, v0
-; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_mov_b32_e32 v6, v0
-; GFX942-NEXT: v_mov_b32_e32 v7, v0
-; GFX942-NEXT: v_mov_b32_e32 v8, v0
-; GFX942-NEXT: v_mov_b32_e32 v9, v0
-; GFX942-NEXT: v_mov_b32_e32 v10, v0
-; GFX942-NEXT: v_mov_b32_e32 v11, v0
-; GFX942-NEXT: v_mov_b32_e32 v12, v0
-; GFX942-NEXT: v_mov_b32_e32 v13, v0
-; GFX942-NEXT: v_mov_b32_e32 v14, v0
-; GFX942-NEXT: v_mov_b32_e32 v15, v0
-; GFX942-NEXT: v_mov_b32_e32 v16, v0
-; GFX942-NEXT: v_mov_b32_e32 v17, v0
-; GFX942-NEXT: v_mov_b32_e32 v18, v0
-; GFX942-NEXT: v_mov_b32_e32 v19, v0
-; GFX942-NEXT: v_mov_b32_e32 v20, v0
-; GFX942-NEXT: v_mov_b32_e32 v21, v0
-; GFX942-NEXT: v_mov_b32_e32 v22, v0
-; GFX942-NEXT: v_mov_b32_e32 v23, v0
-; GFX942-NEXT: v_mov_b32_e32 v24, v0
-; GFX942-NEXT: v_mov_b32_e32 v25, v0
-; GFX942-NEXT: v_mov_b32_e32 v26, v0
-; GFX942-NEXT: v_mov_b32_e32 v27, v0
-; GFX942-NEXT: v_mov_b32_e32 v28, v0
-; GFX942-NEXT: v_mov_b32_e32 v29, v0
-; GFX942-NEXT: v_mov_b32_e32 v30, v0
-; GFX942-NEXT: v_mov_b32_e32 v31, v0
; GFX942-NEXT: .LBB10_1: ; %for.cond.preheader
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_nop 1
@@ -2650,40 +2745,40 @@ define <32 x float> @test_mfma_loop_non_splat_ret_use() #0 {
; GFX908-LABEL: test_mfma_loop_non_splat_ret_use:
; GFX908: ; %bb.0: ; %entry
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_accvgpr_write_b32 a0, 0
; GFX908-NEXT: v_accvgpr_write_b32 a1, 1.0
-; GFX908-NEXT: v_accvgpr_write_b32 a31, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a30, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a29, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a28, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a27, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a26, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a25, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a24, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a23, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a22, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a21, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a20, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a19, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a18, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a17, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a16, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a15, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a14, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a13, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a12, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a11, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a10, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a9, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a8, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a7, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a6, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a5, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a4, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a3, 0
; GFX908-NEXT: v_accvgpr_write_b32 a2, 0
-; GFX908-NEXT: v_accvgpr_write_b32 a0, 0
-; GFX908-NEXT: v_mov_b32_e32 v0, 1.0
+; GFX908-NEXT: v_accvgpr_write_b32 a3, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a4, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a5, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a6, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a7, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a8, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a9, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a10, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a11, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a12, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a13, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a14, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a15, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a16, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a17, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a18, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a19, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a20, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a21, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a22, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a23, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a24, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a25, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a26, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a27, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a28, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a29, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a30, 0
+; GFX908-NEXT: v_accvgpr_write_b32 a31, 0
; GFX908-NEXT: s_mov_b32 s4, 16
+; GFX908-NEXT: v_mov_b32_e32 v0, 1.0
; GFX908-NEXT: v_mov_b32_e32 v1, 2.0
; GFX908-NEXT: .LBB11_1: ; %for.cond.preheader
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -2732,40 +2827,40 @@ define <32 x float> @test_mfma_loop_non_splat_ret_use() #0 {
; GFX90A: ; %bb.0: ; %entry
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v32, 1.0
-; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_mov_b32 s4, 16
+; GFX90A-NEXT: v_mov_b32_e32 v0, 0
+; GFX90A-NEXT: v_mov_b32_e32 v2, 0
+; GFX90A-NEXT: v_mov_b32_e32 v3, 0
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0
+; GFX90A-NEXT: v_mov_b32_e32 v6, 0
+; GFX90A-NEXT: v_mov_b32_e32 v7, 0
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0
+; GFX90A-NEXT: v_mov_b32_e32 v9, 0
+; GFX90A-NEXT: v_mov_b32_e32 v10, 0
+; GFX90A-NEXT: v_mov_b32_e32 v11, 0
+; GFX90A-NEXT: v_mov_b32_e32 v12, 0
+; GFX90A-NEXT: v_mov_b32_e32 v13, 0
+; GFX90A-NEXT: v_mov_b32_e32 v14, 0
+; GFX90A-NEXT: v_mov_b32_e32 v15, 0
+; GFX90A-NEXT: v_mov_b32_e32 v16, 0
+; GFX90A-NEXT: v_mov_b32_e32 v17, 0
+; GFX90A-NEXT: v_mov_b32_e32 v18, 0
+; GFX90A-NEXT: v_mov_b32_e32 v19, 0
+; GFX90A-NEXT: v_mov_b32_e32 v20, 0
+; GFX90A-NEXT: v_mov_b32_e32 v21, 0
+; GFX90A-NEXT: v_mov_b32_e32 v22, 0
+; GFX90A-NEXT: v_mov_b32_e32 v23, 0
+; GFX90A-NEXT: v_mov_b32_e32 v24, 0
+; GFX90A-NEXT: v_mov_b32_e32 v25, 0
+; GFX90A-NEXT: v_mov_b32_e32 v26, 0
+; GFX90A-NEXT: v_mov_b32_e32 v27, 0
+; GFX90A-NEXT: v_mov_b32_e32 v28, 0
+; GFX90A-NEXT: v_mov_b32_e32 v29, 0
+; GFX90A-NEXT: v_mov_b32_e32 v30, 0
+; GFX90A-NEXT: v_mov_b32_e32 v31, 0
; GFX90A-NEXT: v_mov_b32_e32 v33, 2.0
; GFX90A-NEXT: v_mov_b32_e32 v1, v32
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: v_mov_b32_e32 v4, v0
-; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_mov_b32_e32 v6, v0
-; GFX90A-NEXT: v_mov_b32_e32 v7, v0
-; GFX90A-NEXT: v_mov_b32_e32 v8, v0
-; GFX90A-NEXT: v_mov_b32_e32 v9, v0
-; GFX90A-NEXT: v_mov_b32_e32 v10, v0
-; GFX90A-NEXT: v_mov_b32_e32 v11, v0
-; GFX90A-NEXT: v_mov_b32_e32 v12, v0
-; GFX90A-NEXT: v_mov_b32_e32 v13, v0
-; GFX90A-NEXT: v_mov_b32_e32 v14, v0
-; GFX90A-NEXT: v_mov_b32_e32 v15, v0
-; GFX90A-NEXT: v_mov_b32_e32 v16, v0
-; GFX90A-NEXT: v_mov_b32_e32 v17, v0
-; GFX90A-NEXT: v_mov_b32_e32 v18, v0
-; GFX90A-NEXT: v_mov_b32_e32 v19, v0
-; GFX90A-NEXT: v_mov_b32_e32 v20, v0
-; GFX90A-NEXT: v_mov_b32_e32 v21, v0
-; GFX90A-NEXT: v_mov_b32_e32 v22, v0
-; GFX90A-NEXT: v_mov_b32_e32 v23, v0
-; GFX90A-NEXT: v_mov_b32_e32 v24, v0
-; GFX90A-NEXT: v_mov_b32_e32 v25, v0
-; GFX90A-NEXT: v_mov_b32_e32 v26, v0
-; GFX90A-NEXT: v_mov_b32_e32 v27, v0
-; GFX90A-NEXT: v_mov_b32_e32 v28, v0
-; GFX90A-NEXT: v_mov_b32_e32 v29, v0
-; GFX90A-NEXT: v_mov_b32_e32 v30, v0
-; GFX90A-NEXT: v_mov_b32_e32 v31, v0
; GFX90A-NEXT: .LBB11_1: ; %for.cond.preheader
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_nop 1
@@ -2780,40 +2875,40 @@ define <32 x float> @test_mfma_loop_non_splat_ret_use() #0 {
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v32, 1.0
-; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_mov_b32 s0, 16
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-NEXT: v_mov_b32_e32 v3, 0
+; GFX942-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-NEXT: v_mov_b32_e32 v5, 0
+; GFX942-NEXT: v_mov_b32_e32 v6, 0
+; GFX942-NEXT: v_mov_b32_e32 v7, 0
+; GFX942-NEXT: v_mov_b32_e32 v8, 0
+; GFX942-NEXT: v_mov_b32_e32 v9, 0
+; GFX942-NEXT: v_mov_b32_e32 v10, 0
+; GFX942-NEXT: v_mov_b32_e32 v11, 0
+; GFX942-NEXT: v_mov_b32_e32 v12, 0
+; GFX942-NEXT: v_mov_b32_e32 v13, 0
+; GFX942-NEXT: v_mov_b32_e32 v14, 0
+; GFX942-NEXT: v_mov_b32_e32 v15, 0
+; GFX942-NEXT: v_mov_b32_e32 v16, 0
+; GFX942-NEXT: v_mov_b32_e32 v17, 0
+; GFX942-NEXT: v_mov_b32_e32 v18, 0
+; GFX942-NEXT: v_mov_b32_e32 v19, 0
+; GFX942-NEXT: v_mov_b32_e32 v20, 0
+; GFX942-NEXT: v_mov_b32_e32 v21, 0
+; GFX942-NEXT: v_mov_b32_e32 v22, 0
+; GFX942-NEXT: v_mov_b32_e32 v23, 0
+; GFX942-NEXT: v_mov_b32_e32 v24, 0
+; GFX942-NEXT: v_mov_b32_e32 v25, 0
+; GFX942-NEXT: v_mov_b32_e32 v26, 0
+; GFX942-NEXT: v_mov_b32_e32 v27, 0
+; GFX942-NEXT: v_mov_b32_e32 v28, 0
+; GFX942-NEXT: v_mov_b32_e32 v29, 0
+; GFX942-NEXT: v_mov_b32_e32 v30, 0
+; GFX942-NEXT: v_mov_b32_e32 v31, 0
; GFX942-NEXT: v_mov_b32_e32 v33, 2.0
; GFX942-NEXT: v_mov_b32_e32 v1, v32
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: v_mov_b32_e32 v4, v0
-; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_mov_b32_e32 v6, v0
-; GFX942-NEXT: v_mov_b32_e32 v7, v0
-; GFX942-NEXT: v_mov_b32_e32 v8, v0
-; GFX942-NEXT: v_mov_b32_e32 v9, v0
-; GFX942-NEXT: v_mov_b32_e32 v10, v0
-; GFX942-NEXT: v_mov_b32_e32 v11, v0
-; GFX942-NEXT: v_mov_b32_e32 v12, v0
-; GFX942-NEXT: v_mov_b32_e32 v13, v0
-; GFX942-NEXT: v_mov_b32_e32 v14, v0
-; GFX942-NEXT: v_mov_b32_e32 v15, v0
-; GFX942-NEXT: v_mov_b32_e32 v16, v0
-; GFX942-NEXT: v_mov_b32_e32 v17, v0
-; GFX942-NEXT: v_mov_b32_e32 v18, v0
-; GFX942-NEXT: v_mov_b32_e32 v19, v0
-; GFX942-NEXT: v_mov_b32_e32 v20, v0
-; GFX942-NEXT: v_mov_b32_e32 v21, v0
-; GFX942-NEXT: v_mov_b32_e32 v22, v0
-; GFX942-NEXT: v_mov_b32_e32 v23, v0
-; GFX942-NEXT: v_mov_b32_e32 v24, v0
-; GFX942-NEXT: v_mov_b32_e32 v25, v0
-; GFX942-NEXT: v_mov_b32_e32 v26, v0
-; GFX942-NEXT: v_mov_b32_e32 v27, v0
-; GFX942-NEXT: v_mov_b32_e32 v28, v0
-; GFX942-NEXT: v_mov_b32_e32 v29, v0
-; GFX942-NEXT: v_mov_b32_e32 v30, v0
-; GFX942-NEXT: v_mov_b32_e32 v31, v0
; GFX942-NEXT: .LBB11_1: ; %for.cond.preheader
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_nop 1
diff --git a/llvm/test/CodeGen/AMDGPU/mmra.ll b/llvm/test/CodeGen/AMDGPU/mmra.ll
index f241a6dbcfb9f..d5d78ed5c492e 100644
--- a/llvm/test/CodeGen/AMDGPU/mmra.ll
+++ b/llvm/test/CodeGen/AMDGPU/mmra.ll
@@ -156,7 +156,8 @@ define void @cmpxchg(ptr %ptr) {
; CHECK-NEXT: [[FLAT_ATOMIC_CMPSWAP_RTN:%[0-9]+]]:vgpr_32 = FLAT_ATOMIC_CMPSWAP_RTN [[COPY4]], killed [[REG_SEQUENCE2]], 0, 1, implicit $exec, implicit $flat_scr, mmra !1 :: (load store acquire acquire (s32) on %ir.AlignedAddr)
; CHECK-NEXT: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 [[FLAT_ATOMIC_CMPSWAP_RTN]], [[PHI2]], implicit $exec
; CHECK-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64 = S_MOV_B64 -1
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[DEF1]]
; CHECK-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64 = S_OR_B64 [[PHI]], $exec, implicit-def dead $scc
; CHECK-NEXT: [[SI_IF:%[0-9]+]]:sreg_64 = SI_IF killed [[V_CMP_NE_U32_e64_]], %bb.3, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.2
@@ -174,10 +175,10 @@ define void @cmpxchg(ptr %ptr) {
; CHECK-NEXT: successors: %bb.4(0x04000000), %bb.1(0x7c000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[PHI3:%[0-9]+]]:sreg_64 = PHI [[S_OR_B64_]], %bb.1, [[S_OR_B64_1]], %bb.2
- ; CHECK-NEXT: [[PHI4:%[0-9]+]]:vgpr_32 = PHI [[DEF1]], %bb.1, [[V_AND_B32_e64_3]], %bb.2
+ ; CHECK-NEXT: [[PHI4:%[0-9]+]]:vgpr_32 = PHI [[COPY5]], %bb.1, [[V_AND_B32_e64_3]], %bb.2
; CHECK-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:sreg_64 = COPY [[PHI3]]
- ; CHECK-NEXT: [[SI_IF_BREAK:%[0-9]+]]:sreg_64 = SI_IF_BREAK [[COPY5]], [[PHI1]], implicit-def dead $scc
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:sreg_64 = COPY [[PHI3]]
+ ; CHECK-NEXT: [[SI_IF_BREAK:%[0-9]+]]:sreg_64 = SI_IF_BREAK [[COPY6]], [[PHI1]], implicit-def dead $scc
; CHECK-NEXT: SI_LOOP [[SI_IF_BREAK]], %bb.1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.4
; CHECK-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/multilevel-break.ll b/llvm/test/CodeGen/AMDGPU/multilevel-break.ll
index 0c5b90d9c2ae0..ee93ff6933d88 100644
--- a/llvm/test/CodeGen/AMDGPU/multilevel-break.ll
+++ b/llvm/test/CodeGen/AMDGPU/multilevel-break.ll
@@ -45,8 +45,8 @@ define amdgpu_vs void @multi_else_break(<4 x float> %vec, i32 %ub, i32 %cont) {
;
; GCN-LABEL: multi_else_break:
; GCN: ; %bb.0: ; %main_body
-; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_mov_b64 s[0:1], 0
+; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_branch .LBB0_2
; GCN-NEXT: .LBB0_1: ; %loop.exit.guard
; GCN-NEXT: ; in Loop: Header=BB0_2 Depth=1
diff --git a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
index dec44a092485f..2e4d48b30c1f0 100644
--- a/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
+++ b/llvm/test/CodeGen/AMDGPU/no-fold-accvgpr-mov.ll
@@ -6,9 +6,9 @@ define amdgpu_kernel void @matmul_kernel(i32 %a0, i32 %a1) {
; GFX942-LABEL: matmul_kernel:
; GFX942: ; %bb.0: ; %entry
; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX942-NEXT: v_mov_b32_e32 v1, 0
; GFX942-NEXT: s_mov_b32 s0, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
+; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: v_mov_b32_e32 v1, 0
; GFX942-NEXT: s_mov_b32 s6, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_cmp_lg_u32 s2, 0
diff --git a/llvm/test/CodeGen/AMDGPU/permute.ll b/llvm/test/CodeGen/AMDGPU/permute.ll
index 50392ecd3e4e4..2dc526c2a0a45 100644
--- a/llvm/test/CodeGen/AMDGPU/permute.ll
+++ b/llvm/test/CodeGen/AMDGPU/permute.ll
@@ -435,16 +435,16 @@ bb:
define amdgpu_kernel void @dword_from_v16i16_combine_loop(ptr addrspace(1) %out) {
; GCN-LABEL: dword_from_v16i16_combine_loop:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: v_and_b32_e32 v1, 3, v0
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GCN-NEXT: v_mov_b32_e32 v0, 0
+; GCN-NEXT: v_and_b32_e32 v0, 3, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GCN-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GCN-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v2, s1
-; GCN-NEXT: v_mov_b32_e32 v1, s0
-; GCN-NEXT: flat_store_byte v[1:2], v0
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: flat_store_byte v[0:1], v2
; GCN-NEXT: s_endpgm
entry:
%tid = tail call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/phi-av-pressure.ll b/llvm/test/CodeGen/AMDGPU/phi-av-pressure.ll
index a4bf3705534bc..1094b76d01a90 100644
--- a/llvm/test/CodeGen/AMDGPU/phi-av-pressure.ll
+++ b/llvm/test/CodeGen/AMDGPU/phi-av-pressure.ll
@@ -17,7 +17,7 @@ define amdgpu_kernel void @main(i1 %arg, ptr %ptr, ptr addrspace(1) %ptr1, ptr a
; GFX950-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x2c
; GFX950-NEXT: s_mov_b32 s12, s8
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: scratch_load_dwordx4 v[40:43], off, s33
+; GFX950-NEXT: scratch_load_dwordx4 v[42:45], off, s33
; GFX950-NEXT: s_mov_b32 s13, s9
; GFX950-NEXT: s_bitcmp1_b32 s6, 0
; GFX950-NEXT: s_cselect_b64 s[34:35], -1, 0
@@ -30,67 +30,68 @@ define amdgpu_kernel void @main(i1 %arg, ptr %ptr, ptr addrspace(1) %ptr1, ptr a
; GFX950-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX950-NEXT: v_mov_b32_e32 v31, v0
; GFX950-NEXT: s_mov_b32 s32, 0
+; GFX950-NEXT: v_mov_b32_e32 v41, 0
; GFX950-NEXT: ; implicit-def: $sgpr15
; GFX950-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GFX950-NEXT: v_mov_b32_e32 v1, 0
-; GFX950-NEXT: v_lshl_add_u64 v[10:11], v[0:1], 3, s[38:39]
-; GFX950-NEXT: global_load_dwordx4 v[2:5], v[10:11], off
-; GFX950-NEXT: global_load_dwordx4 v[6:9], v1, s[38:39]
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
-; GFX950-NEXT: v_mov_b64_e32 v[10:11], s[36:37]
+; GFX950-NEXT: v_mov_b32_e32 v40, v0
+; GFX950-NEXT: v_lshl_add_u64 v[8:9], v[40:41], 3, s[38:39]
+; GFX950-NEXT: global_load_dwordx4 v[0:3], v[8:9], off
+; GFX950-NEXT: global_load_dwordx4 v[4:7], v41, s[38:39]
+; GFX950-NEXT: s_mov_b32 s1, 0
+; GFX950-NEXT: v_mov_b64_e32 v[8:9], s[36:37]
; GFX950-NEXT: s_and_b64 vcc, exec, s[34:35]
-; GFX950-NEXT: v_mov_b32_e32 v12, v1
-; GFX950-NEXT: v_mov_b32_e32 v14, v1
-; GFX950-NEXT: v_mov_b32_e32 v15, v1
-; GFX950-NEXT: v_mov_b32_e32 v16, v1
-; GFX950-NEXT: v_mov_b32_e32 v18, v1
-; GFX950-NEXT: v_mov_b32_e32 v17, v1
-; GFX950-NEXT: v_mov_b32_e32 v19, v1
-; GFX950-NEXT: v_mov_b32_e32 v20, v1
-; GFX950-NEXT: v_mov_b32_e32 v21, v1
+; GFX950-NEXT: v_mov_b32_e32 v40, v41
+; GFX950-NEXT: s_mov_b32 s0, s1
+; GFX950-NEXT: v_mov_b32_e32 v10, v41
+; GFX950-NEXT: v_mov_b32_e32 v11, v41
+; GFX950-NEXT: v_mov_b32_e32 v12, v41
+; GFX950-NEXT: v_mov_b32_e32 v14, v41
+; GFX950-NEXT: v_mov_b32_e32 v13, v41
+; GFX950-NEXT: v_mov_b32_e32 v15, v41
+; GFX950-NEXT: v_mov_b32_e32 v16, v41
+; GFX950-NEXT: v_mov_b32_e32 v17, v41
; GFX950-NEXT: .LBB0_1: ; %bb4
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_mov_b32_e32 v13, v1
-; GFX950-NEXT: v_lshlrev_b64 v[22:23], 3, v[12:13]
-; GFX950-NEXT: v_lshl_add_u64 v[22:23], s[38:39], 0, v[22:23]
-; GFX950-NEXT: global_load_dwordx4 v[22:25], v[22:23], off
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: v_lshl_add_u64 v[26:27], v[0:1], 3, s[38:39]
+; GFX950-NEXT: s_lshl_b64 s[2:3], s[0:1], 3
+; GFX950-NEXT: s_add_u32 s2, s38, s2
+; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_lshl_add_u64 v[18:19], v[40:41], 3, s[38:39]
+; GFX950-NEXT: s_addc_u32 s3, s39, s3
+; GFX950-NEXT: global_load_dwordx4 v[18:21], v[18:19], off
; GFX950-NEXT: s_waitcnt vmcnt(1)
-; GFX950-NEXT: v_or_b32_e32 v0, v40, v6
-; GFX950-NEXT: v_or_b32_e32 v13, v41, v7
-; GFX950-NEXT: v_or_b32_e32 v30, v42, v8
-; GFX950-NEXT: v_or_b32_e32 v31, v43, v9
-; GFX950-NEXT: global_load_dwordx4 v[26:29], v[26:27], off
-; GFX950-NEXT: v_or_b32_e32 v18, v5, v18
-; GFX950-NEXT: v_or_b32_e32 v16, v4, v16
-; GFX950-NEXT: v_or_b32_e32 v15, v3, v15
-; GFX950-NEXT: v_or_b32_e32 v14, v2, v14
-; GFX950-NEXT: v_or_b32_e32 v12, 1, v12
-; GFX950-NEXT: v_mov_b32_e32 v40, 0
-; GFX950-NEXT: v_mov_b32_e32 v41, 0
+; GFX950-NEXT: v_or_b32_e32 v26, v42, v4
+; GFX950-NEXT: global_load_dwordx4 v[22:25], v41, s[2:3]
+; GFX950-NEXT: v_or_b32_e32 v27, v43, v5
+; GFX950-NEXT: v_or_b32_e32 v28, v44, v6
+; GFX950-NEXT: v_or_b32_e32 v29, v45, v7
+; GFX950-NEXT: v_or_b32_e32 v14, v3, v14
+; GFX950-NEXT: v_or_b32_e32 v12, v2, v12
+; GFX950-NEXT: v_or_b32_e32 v11, v1, v11
+; GFX950-NEXT: v_or_b32_e32 v10, v0, v10
; GFX950-NEXT: v_mov_b32_e32 v42, 0
; GFX950-NEXT: v_mov_b32_e32 v43, 0
+; GFX950-NEXT: v_mov_b32_e32 v44, 0
+; GFX950-NEXT: v_mov_b32_e32 v45, 0
+; GFX950-NEXT: s_or_b32 s0, s0, 1
; GFX950-NEXT: s_waitcnt vmcnt(1)
-; GFX950-NEXT: v_or_b32_e32 v25, v25, v31
-; GFX950-NEXT: v_or_b32_e32 v24, v24, v30
-; GFX950-NEXT: v_or_b32_e32 v23, v23, v13
-; GFX950-NEXT: v_or_b32_e32 v22, v22, v0
-; GFX950-NEXT: scratch_store_dwordx4 off, v[22:25], s33
-; GFX950-NEXT: flat_load_dword v0, v[10:11]
+; GFX950-NEXT: v_or_b32_e32 v17, v21, v17
+; GFX950-NEXT: v_or_b32_e32 v16, v20, v16
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_or_b32_e32 v21, v29, v21
-; GFX950-NEXT: v_or_b32_e32 v20, v28, v20
-; GFX950-NEXT: v_or_b32_e32 v19, v27, v19
-; GFX950-NEXT: v_or_b32_e32 v17, v26, v17
+; GFX950-NEXT: v_or_b32_e32 v25, v25, v29
+; GFX950-NEXT: v_or_b32_e32 v24, v24, v28
+; GFX950-NEXT: v_or_b32_e32 v23, v23, v27
+; GFX950-NEXT: v_or_b32_e32 v22, v22, v26
+; GFX950-NEXT: scratch_store_dwordx4 off, v[22:25], s33
+; GFX950-NEXT: flat_load_dword v40, v[8:9]
+; GFX950-NEXT: v_or_b32_e32 v15, v19, v15
+; GFX950-NEXT: v_or_b32_e32 v13, v18, v13
; GFX950-NEXT: s_mov_b64 vcc, vcc
; GFX950-NEXT: s_cbranch_vccz .LBB0_1
; GFX950-NEXT: ; %bb.2: ; %bb2
-; GFX950-NEXT: v_or_b32_e32 v3, v21, v18
-; GFX950-NEXT: v_or_b32_e32 v2, v20, v16
-; GFX950-NEXT: v_or_b32_e32 v1, v19, v15
-; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: v_or_b32_e32 v0, v17, v14
+; GFX950-NEXT: v_or_b32_e32 v3, v17, v14
+; GFX950-NEXT: v_or_b32_e32 v2, v16, v12
+; GFX950-NEXT: v_or_b32_e32 v1, v15, v11
+; GFX950-NEXT: v_or_b32_e32 v0, v13, v10
; GFX950-NEXT: scratch_store_dwordx4 off, v[0:3], s33
; GFX950-NEXT: s_endpgm
bb:
diff --git a/llvm/test/CodeGen/AMDGPU/phi-constant-operand-regclass.ll b/llvm/test/CodeGen/AMDGPU/phi-constant-operand-regclass.ll
new file mode 100644
index 0000000000000..fd22bc4a15d66
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/phi-constant-operand-regclass.ll
@@ -0,0 +1,227 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -O0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -structurizecfg-skip-uniform-regions=1 -stop-after=finalize-isel < %s | FileCheck %s -check-prefix=GFX900
+; RUN: llc -O0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -structurizecfg-skip-uniform-regions=1 -stop-after=finalize-isel < %s | FileCheck %s -check-prefix=GFX90A
+
+; Regression tests for cases where uniform PHIs were moved into VGPRs, which
+; failed MIR verification because SI_IF_BREAK requires its operands to have an
+; SGPR class.
+
+define i64 @if_break_mask_shares_constant_with_divergent_phi(i1 %cond) {
+ ; GFX900-LABEL: name: if_break_mask_shares_constant_with_divergent_phi
+ ; GFX900: bb.0.entry:
+ ; GFX900-NEXT: successors: %bb.3(0x40000000), %bb.1(0x40000000)
+ ; GFX900-NEXT: liveins: $vgpr0
+ ; GFX900-NEXT: {{ $}}
+ ; GFX900-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX900-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 1, [[COPY]], implicit $exec
+ ; GFX900-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 killed [[V_AND_B32_e64_]], 1, implicit $exec
+ ; GFX900-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX900-NEXT: [[COPY1:%[0-9]+]]:sreg_64 = COPY [[S_MOV_B64_]]
+ ; GFX900-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY [[V_CMP_EQ_U32_e64_]]
+ ; GFX900-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX900-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 $exec, killed [[S_MOV_B64_1]], implicit-def dead $scc
+ ; GFX900-NEXT: $vcc = COPY [[S_AND_B64_]]
+ ; GFX900-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY [[COPY1]], implicit $exec
+ ; GFX900-NEXT: S_CBRANCH_VCCNZ %bb.3, implicit $vcc
+ ; GFX900-NEXT: S_BRANCH %bb.1
+ ; GFX900-NEXT: {{ $}}
+ ; GFX900-NEXT: bb.1.loop:
+ ; GFX900-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; GFX900-NEXT: {{ $}}
+ ; GFX900-NEXT: [[PHI:%[0-9]+]]:sreg_64 = PHI [[COPY1]], %bb.0, %2, %bb.1
+ ; GFX900-NEXT: [[PHI1:%[0-9]+]]:sreg_64 = PHI [[S_MOV_B64_]], %bb.0, %23, %bb.1
+ ; GFX900-NEXT: [[COPY4:%[0-9]+]]:sreg_64 = COPY [[COPY2]]
+ ; GFX900-NEXT: [[SI_IF_BREAK:%[0-9]+]]:sreg_64 = SI_IF_BREAK [[COPY4]], [[PHI]], implicit-def dead $scc
+ ; GFX900-NEXT: [[S_MOV_B64_2:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX900-NEXT: [[COPY5:%[0-9]+]]:vreg_64 = COPY [[PHI1]], implicit $exec
+ ; GFX900-NEXT: SI_LOOP [[SI_IF_BREAK]], %bb.1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ ; GFX900-NEXT: S_BRANCH %bb.2
+ ; GFX900-NEXT: {{ $}}
+ ; GFX900-NEXT: bb.2.done:
+ ; GFX900-NEXT: successors: %bb.3(0x80000000)
+ ; GFX900-NEXT: {{ $}}
+ ; GFX900-NEXT: [[PHI2:%[0-9]+]]:vreg_64 = PHI [[COPY5]], %bb.1
+ ; GFX900-NEXT: [[PHI3:%[0-9]+]]:sreg_64 = PHI [[SI_IF_BREAK]], %bb.1
+ ; GFX900-NEXT: SI_END_CF [[PHI3]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ ; GFX900-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[PHI2]]
+ ; GFX900-NEXT: S_BRANCH %bb.3
+ ; GFX900-NEXT: {{ $}}
+ ; GFX900-NEXT: bb.3.UnifiedReturnBlock:
+ ; GFX900-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI [[COPY3]], %bb.0, [[COPY6]], %bb.2
+ ; GFX900-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32
+ ; GFX900-NEXT: [[V_LSHRREV_B64_e64_:%[0-9]+]]:vreg_64 = V_LSHRREV_B64_e64 killed [[S_MOV_B32_]], [[PHI4]], implicit $exec
+ ; GFX900-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[V_LSHRREV_B64_e64_]].sub0
+ ; GFX900-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[PHI4]].sub0
+ ; GFX900-NEXT: $vgpr0 = COPY [[COPY8]]
+ ; GFX900-NEXT: $vgpr1 = COPY [[COPY7]]
+ ; GFX900-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
+ ;
+ ; GFX90A-LABEL: name: if_break_mask_shares_constant_with_divergent_phi
+ ; GFX90A: bb.0.entry:
+ ; GFX90A-NEXT: successors: %bb.3(0x40000000), %bb.1(0x40000000)
+ ; GFX90A-NEXT: liveins: $vgpr0
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX90A-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 1, [[COPY]], implicit $exec
+ ; GFX90A-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 killed [[V_AND_B32_e64_]], 1, implicit $exec
+ ; GFX90A-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX90A-NEXT: [[COPY1:%[0-9]+]]:sreg_64 = COPY [[S_MOV_B64_]]
+ ; GFX90A-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY [[V_CMP_EQ_U32_e64_]]
+ ; GFX90A-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX90A-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 $exec, killed [[S_MOV_B64_1]], implicit-def dead $scc
+ ; GFX90A-NEXT: $vcc = COPY [[S_AND_B64_]]
+ ; GFX90A-NEXT: [[COPY3:%[0-9]+]]:vreg_64_align2 = COPY [[COPY1]], implicit $exec
+ ; GFX90A-NEXT: S_CBRANCH_VCCNZ %bb.3, implicit $vcc
+ ; GFX90A-NEXT: S_BRANCH %bb.1
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: bb.1.loop:
+ ; GFX90A-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: [[PHI:%[0-9]+]]:sreg_64 = PHI [[COPY1]], %bb.0, %2, %bb.1
+ ; GFX90A-NEXT: [[PHI1:%[0-9]+]]:sreg_64 = PHI [[S_MOV_B64_]], %bb.0, %23, %bb.1
+ ; GFX90A-NEXT: [[COPY4:%[0-9]+]]:sreg_64 = COPY [[COPY2]]
+ ; GFX90A-NEXT: [[SI_IF_BREAK:%[0-9]+]]:sreg_64 = SI_IF_BREAK [[COPY4]], [[PHI]], implicit-def dead $scc
+ ; GFX90A-NEXT: [[S_MOV_B64_2:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX90A-NEXT: [[COPY5:%[0-9]+]]:av_64_align2 = COPY [[PHI1]], implicit $exec
+ ; GFX90A-NEXT: SI_LOOP [[SI_IF_BREAK]], %bb.1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ ; GFX90A-NEXT: S_BRANCH %bb.2
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: bb.2.done:
+ ; GFX90A-NEXT: successors: %bb.3(0x80000000)
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: [[PHI2:%[0-9]+]]:av_64_align2 = PHI [[COPY5]], %bb.1
+ ; GFX90A-NEXT: [[PHI3:%[0-9]+]]:sreg_64 = PHI [[SI_IF_BREAK]], %bb.1
+ ; GFX90A-NEXT: SI_END_CF [[PHI3]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ ; GFX90A-NEXT: [[COPY6:%[0-9]+]]:av_64_align2 = COPY [[PHI2]]
+ ; GFX90A-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2 = COPY [[COPY6]], implicit $exec
+ ; GFX90A-NEXT: S_BRANCH %bb.3
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: bb.3.UnifiedReturnBlock:
+ ; GFX90A-NEXT: [[PHI4:%[0-9]+]]:vreg_64_align2 = PHI [[COPY3]], %bb.0, [[COPY7]], %bb.2
+ ; GFX90A-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32
+ ; GFX90A-NEXT: [[V_LSHRREV_B64_e64_:%[0-9]+]]:vreg_64_align2 = V_LSHRREV_B64_e64 killed [[S_MOV_B32_]], [[PHI4]], implicit $exec
+ ; GFX90A-NEXT: [[COPY8:%[0-9]+]]:av_32 = COPY [[V_LSHRREV_B64_e64_]].sub0
+ ; GFX90A-NEXT: [[COPY9:%[0-9]+]]:av_32 = COPY [[PHI4]].sub0
+ ; GFX90A-NEXT: $vgpr0 = COPY [[COPY9]]
+ ; GFX90A-NEXT: $vgpr1 = COPY [[COPY8]]
+ ; GFX90A-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
+entry:
+ br i1 false, label %join, label %loop
+
+loop:
+ %q = phi ptr [ addrspacecast (ptr addrspace(4) null to ptr), %entry ], [ null, %loop ]
+ br i1 %cond, label %done, label %loop
+
+done:
+ %r = ptrtoaddr ptr %q to i64
+ ret i64 %r
+
+join:
+ ret i64 0
+}
+
+define i64 @divergent_loop_with_bypass(i1 %cond, i64 %val) {
+ ; GFX900-LABEL: name: divergent_loop_with_bypass
+ ; GFX900: bb.0.entry:
+ ; GFX900-NEXT: successors: %bb.3(0x40000000), %bb.1(0x40000000)
+ ; GFX900-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; GFX900-NEXT: {{ $}}
+ ; GFX900-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX900-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX900-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX900-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
+ ; GFX900-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 1, [[COPY2]], implicit $exec
+ ; GFX900-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 killed [[V_AND_B32_e64_]], 1, implicit $exec
+ ; GFX900-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX900-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
+ ; GFX900-NEXT: [[COPY4:%[0-9]+]]:sreg_64 = COPY [[V_CMP_EQ_U32_e64_]]
+ ; GFX900-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX900-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 $exec, killed [[S_MOV_B64_1]], implicit-def dead $scc
+ ; GFX900-NEXT: $vcc = COPY [[S_AND_B64_]]
+ ; GFX900-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 0, implicit $exec
+ ; GFX900-NEXT: S_CBRANCH_VCCNZ %bb.3, implicit $vcc
+ ; GFX900-NEXT: S_BRANCH %bb.1
+ ; GFX900-NEXT: {{ $}}
+ ; GFX900-NEXT: bb.1.loop:
+ ; GFX900-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; GFX900-NEXT: {{ $}}
+ ; GFX900-NEXT: [[PHI:%[0-9]+]]:sreg_64 = PHI [[S_MOV_B64_]], %bb.0, %1, %bb.1
+ ; GFX900-NEXT: [[COPY5:%[0-9]+]]:sreg_64 = COPY [[COPY4]]
+ ; GFX900-NEXT: [[SI_IF_BREAK:%[0-9]+]]:sreg_64 = SI_IF_BREAK [[COPY5]], [[PHI]], implicit-def dead $scc
+ ; GFX900-NEXT: SI_LOOP [[SI_IF_BREAK]], %bb.1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ ; GFX900-NEXT: S_BRANCH %bb.2
+ ; GFX900-NEXT: {{ $}}
+ ; GFX900-NEXT: bb.2.Flow:
+ ; GFX900-NEXT: successors: %bb.3(0x80000000)
+ ; GFX900-NEXT: {{ $}}
+ ; GFX900-NEXT: [[PHI1:%[0-9]+]]:sreg_64 = PHI [[SI_IF_BREAK]], %bb.1
+ ; GFX900-NEXT: SI_END_CF [[PHI1]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ ; GFX900-NEXT: S_BRANCH %bb.3
+ ; GFX900-NEXT: {{ $}}
+ ; GFX900-NEXT: bb.3.join:
+ ; GFX900-NEXT: [[PHI2:%[0-9]+]]:vreg_64 = PHI [[V_MOV_B]], %bb.0, [[COPY3]], %bb.2
+ ; GFX900-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32
+ ; GFX900-NEXT: [[V_LSHRREV_B64_e64_:%[0-9]+]]:vreg_64 = V_LSHRREV_B64_e64 killed [[S_MOV_B32_]], [[PHI2]], implicit $exec
+ ; GFX900-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[V_LSHRREV_B64_e64_]].sub0
+ ; GFX900-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[PHI2]].sub0
+ ; GFX900-NEXT: $vgpr0 = COPY [[COPY7]]
+ ; GFX900-NEXT: $vgpr1 = COPY [[COPY6]]
+ ; GFX900-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
+ ;
+ ; GFX90A-LABEL: name: divergent_loop_with_bypass
+ ; GFX90A: bb.0.entry:
+ ; GFX90A-NEXT: successors: %bb.3(0x40000000), %bb.1(0x40000000)
+ ; GFX90A-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX90A-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX90A-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX90A-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
+ ; GFX90A-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 1, [[COPY2]], implicit $exec
+ ; GFX90A-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 killed [[V_AND_B32_e64_]], 1, implicit $exec
+ ; GFX90A-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX90A-NEXT: [[COPY3:%[0-9]+]]:av_64_align2 = COPY [[REG_SEQUENCE]]
+ ; GFX90A-NEXT: [[COPY4:%[0-9]+]]:sreg_64 = COPY [[V_CMP_EQ_U32_e64_]]
+ ; GFX90A-NEXT: [[S_MOV_B64_1:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+ ; GFX90A-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 $exec, killed [[S_MOV_B64_1]], implicit-def dead $scc
+ ; GFX90A-NEXT: $vcc = COPY [[S_AND_B64_]]
+ ; GFX90A-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: S_CBRANCH_VCCNZ %bb.3, implicit $vcc
+ ; GFX90A-NEXT: S_BRANCH %bb.1
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: bb.1.loop:
+ ; GFX90A-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: [[PHI:%[0-9]+]]:sreg_64 = PHI [[S_MOV_B64_]], %bb.0, %1, %bb.1
+ ; GFX90A-NEXT: [[COPY5:%[0-9]+]]:sreg_64 = COPY [[COPY4]]
+ ; GFX90A-NEXT: [[SI_IF_BREAK:%[0-9]+]]:sreg_64 = SI_IF_BREAK [[COPY5]], [[PHI]], implicit-def dead $scc
+ ; GFX90A-NEXT: SI_LOOP [[SI_IF_BREAK]], %bb.1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ ; GFX90A-NEXT: S_BRANCH %bb.2
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: bb.2.Flow:
+ ; GFX90A-NEXT: successors: %bb.3(0x80000000)
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: [[PHI1:%[0-9]+]]:sreg_64 = PHI [[SI_IF_BREAK]], %bb.1
+ ; GFX90A-NEXT: SI_END_CF [[PHI1]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ ; GFX90A-NEXT: [[COPY6:%[0-9]+]]:vreg_64_align2 = COPY [[COPY3]], implicit $exec
+ ; GFX90A-NEXT: S_BRANCH %bb.3
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: bb.3.join:
+ ; GFX90A-NEXT: [[PHI2:%[0-9]+]]:vreg_64_align2 = PHI [[V_MOV_B]], %bb.0, [[COPY6]], %bb.2
+ ; GFX90A-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32
+ ; GFX90A-NEXT: [[V_LSHRREV_B64_e64_:%[0-9]+]]:vreg_64_align2 = V_LSHRREV_B64_e64 killed [[S_MOV_B32_]], [[PHI2]], implicit $exec
+ ; GFX90A-NEXT: [[COPY7:%[0-9]+]]:av_32 = COPY [[V_LSHRREV_B64_e64_]].sub0
+ ; GFX90A-NEXT: [[COPY8:%[0-9]+]]:av_32 = COPY [[PHI2]].sub0
+ ; GFX90A-NEXT: $vgpr0 = COPY [[COPY8]]
+ ; GFX90A-NEXT: $vgpr1 = COPY [[COPY7]]
+ ; GFX90A-NEXT: SI_RETURN implicit $vgpr0, implicit $vgpr1
+entry:
+ br i1 false, label %join, label %loop
+
+loop:
+ br i1 %cond, label %join, label %loop
+
+join:
+ %r = phi i64 [ 0, %entry ], [ %val, %loop ]
+ ret i64 %r
+}
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
index 5027c3da09461..162fdba95a36d 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
@@ -380,9 +380,9 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1) %buffer) {
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x2800, v0
; GFX8-NEXT: s_movk_i32 s0, 0x2800
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_mov_b32_e32 v13, 0x7f
; GFX8-NEXT: v_mov_b32_e32 v6, 0
; GFX8-NEXT: v_mov_b32_e32 v7, 0
-; GFX8-NEXT: v_mov_b32_e32 v13, 0x7f
; GFX8-NEXT: s_movk_i32 s1, 0x800
; GFX8-NEXT: s_movk_i32 s2, 0x1000
; GFX8-NEXT: s_movk_i32 s3, 0x1800
@@ -506,9 +506,9 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1) %buffer) {
; GFX900-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX900-NEXT: v_add_co_u32_e32 v0, vcc, 0x2800, v0
; GFX900-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX900-NEXT: v_mov_b32_e32 v7, 0x7f
; GFX900-NEXT: v_mov_b32_e32 v4, 0
; GFX900-NEXT: v_mov_b32_e32 v5, 0
-; GFX900-NEXT: v_mov_b32_e32 v7, 0x7f
; GFX900-NEXT: s_movk_i32 s2, 0xf000
; GFX900-NEXT: s_movk_i32 s3, 0x1000
; GFX900-NEXT: s_movk_i32 s4, 0x2000
@@ -730,8 +730,8 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1) %buffer) {
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 0x2800, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], 0, 0
; GFX90A-NEXT: v_mov_b32_e32 v17, 0x7f
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], 0, 0
; GFX90A-NEXT: s_movk_i32 s2, 0xf000
; GFX90A-NEXT: s_movk_i32 s3, 0x1000
; GFX90A-NEXT: s_movk_i32 s4, 0x2000
diff --git a/llvm/test/CodeGen/AMDGPU/reg-coalescer-sched-crash.ll b/llvm/test/CodeGen/AMDGPU/reg-coalescer-sched-crash.ll
index fb958bcf8d104..1d895274bab8c 100644
--- a/llvm/test/CodeGen/AMDGPU/reg-coalescer-sched-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/reg-coalescer-sched-crash.ll
@@ -10,9 +10,9 @@ declare i32 @llvm.amdgcn.workitem.id.x() #0
define amdgpu_kernel void @reg_coalescer_breaks_dead(ptr addrspace(1) nocapture readonly %arg, i32 %arg1, i32 %arg2, i32 %arg3, i1 %c0) #1 {
; GFX6-LABEL: reg_coalescer_breaks_dead:
; GFX6: ; %bb.0: ; %bb
-; GFX6-NEXT: v_mov_b32_e32 v1, 0
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX6-NEXT: v_mov_b32_e32 v2, 0
+; GFX6-NEXT: v_mov_b32_e32 v0, 0
+; GFX6-NEXT: v_mov_b32_e32 v1, 0
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB0_2
; GFX6-NEXT: ; %bb.1: ; %bb3
@@ -25,8 +25,8 @@ define amdgpu_kernel void @reg_coalescer_breaks_dead(ptr addrspace(1) nocapture
; GFX6-NEXT: s_addc_u32 s3, s7, s3
; GFX6-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v1, s2
-; GFX6-NEXT: v_mov_b32_e32 v2, s3
+; GFX6-NEXT: v_mov_b32_e32 v0, s2
+; GFX6-NEXT: v_mov_b32_e32 v1, s3
; GFX6-NEXT: .LBB0_2: ; %bb4
; GFX6-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX6-NEXT: s_load_dword s0, s[4:5], 0xe
@@ -35,14 +35,14 @@ define amdgpu_kernel void @reg_coalescer_breaks_dead(ptr addrspace(1) nocapture
; GFX6-NEXT: s_cbranch_scc1 .LBB0_4
; GFX6-NEXT: ; %bb.3: ; %bb15
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_write_b64 v0, v[1:2]
+; GFX6-NEXT: ds_write_b64 v0, v[0:1]
; GFX6-NEXT: .LBB0_4: ; %bb16
;
; GFX8-LABEL: reg_coalescer_breaks_dead:
; GFX8: ; %bb.0: ; %bb
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX8-NEXT: v_mov_b32_e32 v2, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: v_mov_b32_e32 v1, 0
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB0_2
; GFX8-NEXT: ; %bb.1: ; %bb3
@@ -55,8 +55,8 @@ define amdgpu_kernel void @reg_coalescer_breaks_dead(ptr addrspace(1) nocapture
; GFX8-NEXT: s_addc_u32 s3, s7, s3
; GFX8-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_mov_b32_e32 v2, s3
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: .LBB0_2: ; %bb4
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_load_dword s0, s[4:5], 0x38
@@ -65,7 +65,7 @@ define amdgpu_kernel void @reg_coalescer_breaks_dead(ptr addrspace(1) nocapture
; GFX8-NEXT: s_cbranch_scc1 .LBB0_4
; GFX8-NEXT: ; %bb.3: ; %bb15
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: ds_write_b64 v0, v[1:2]
+; GFX8-NEXT: ds_write_b64 v0, v[0:1]
; GFX8-NEXT: .LBB0_4: ; %bb16
bb:
%id.x = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/rem_i128.ll b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
index a52d709ca28cc..a77adf6a545e8 100644
--- a/llvm/test/CodeGen/AMDGPU/rem_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
@@ -906,12 +906,12 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:344 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:348 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:352 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
@@ -921,7 +921,7 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-O0-NEXT: s_mov_b32 s4, 64
; GFX9-O0-NEXT: v_sub_u32_e64 v20, s4, v4
-; GFX9-O0-NEXT: v_lshlrev_b64 v[20:21], v20, v[16:17]
+; GFX9-O0-NEXT: v_lshlrev_b64 v[20:21], v20, v[14:15]
; GFX9-O0-NEXT: v_mov_b32_e32 v22, v21
; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v22
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
@@ -932,7 +932,7 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v22, v7
; GFX9-O0-NEXT: v_cmp_lt_u32_e64 s[6:7], v4, s4
; GFX9-O0-NEXT: v_sub_u32_e64 v5, v4, s4
-; GFX9-O0-NEXT: v_lshrrev_b64 v[20:21], v5, v[16:17]
+; GFX9-O0-NEXT: v_lshrrev_b64 v[20:21], v5, v[14:15]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v21
; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v22, s[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6
@@ -950,46 +950,47 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5]
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5
-; GFX9-O0-NEXT: v_lshrrev_b64 v[4:5], v4, v[16:17]
-; GFX9-O0-NEXT: v_mov_b32_e32 v17, v5
+; GFX9-O0-NEXT: v_lshrrev_b64 v[4:5], v4, v[14:15]
+; GFX9-O0-NEXT: v_mov_b32_e32 v15, v5
; GFX9-O0-NEXT: s_mov_b64 s[4:5], 0
; GFX9-O0-NEXT: s_mov_b32 s8, s5
-; GFX9-O0-NEXT: v_mov_b32_e32 v16, s8
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v16, v16, v17, s[6:7]
+; GFX9-O0-NEXT: v_mov_b32_e32 v14, s8
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v14, v14, v15, s[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v4
; GFX9-O0-NEXT: s_mov_b32 s8, s4
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s8
; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[6:7]
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v5, v16
+; GFX9-O0-NEXT: v_mov_b32_e32 v5, v14
; GFX9-O0-NEXT: s_mov_b64 s[10:11], -1
; GFX9-O0-NEXT: s_mov_b32 s9, s10
-; GFX9-O0-NEXT: v_mov_b32_e32 v16, v14
-; GFX9-O0-NEXT: v_add_co_u32_e64 v18, s[6:7], v16, s9
-; GFX9-O0-NEXT: s_mov_b32 s8, s11
-; GFX9-O0-NEXT: v_mov_b32_e32 v14, v15
-; GFX9-O0-NEXT: v_mov_b32_e32 v15, s8
-; GFX9-O0-NEXT: v_addc_co_u32_e64 v14, s[6:7], v14, v15, s[6:7]
-; GFX9-O0-NEXT: ; kill: def $vgpr18 killed $vgpr18 def $vgpr18_vgpr19 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v19, v14
; GFX9-O0-NEXT: v_mov_b32_e32 v14, v12
-; GFX9-O0-NEXT: v_mov_b32_e32 v15, s9
-; GFX9-O0-NEXT: v_addc_co_u32_e64 v16, s[6:7], v14, v15, s[6:7]
+; GFX9-O0-NEXT: v_add_co_u32_e64 v14, s[6:7], v14, s9
+; GFX9-O0-NEXT: s_mov_b32 s8, s11
; GFX9-O0-NEXT: v_mov_b32_e32 v12, v13
; GFX9-O0-NEXT: v_mov_b32_e32 v13, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v12, s[6:7], v12, v13, s[6:7]
-; GFX9-O0-NEXT: ; kill: def $vgpr16 killed $vgpr16 def $vgpr16_vgpr17 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v17, v12
-; GFX9-O0-NEXT: v_mov_b32_e32 v13, s5
-; GFX9-O0-NEXT: v_mov_b32_e32 v12, s4
-; GFX9-O0-NEXT: v_mov_b32_e32 v15, s5
-; GFX9-O0-NEXT: v_mov_b32_e32 v14, s4
-; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec
+; GFX9-O0-NEXT: v_mov_b32_e32 v15, v12
+; GFX9-O0-NEXT: v_mov_b32_e32 v12, v16
+; GFX9-O0-NEXT: v_mov_b32_e32 v13, s9
+; GFX9-O0-NEXT: v_addc_co_u32_e64 v12, s[6:7], v12, v13, s[6:7]
+; GFX9-O0-NEXT: v_mov_b32_e32 v13, v17
+; GFX9-O0-NEXT: v_mov_b32_e32 v16, s8
+; GFX9-O0-NEXT: v_addc_co_u32_e64 v16, s[6:7], v13, v16, s[6:7]
+; GFX9-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
+; GFX9-O0-NEXT: v_mov_b32_e32 v13, v16
+; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:316 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
-; GFX9-O0-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:320 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:308 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
-; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:312 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: v_mov_b32_e32 v15, s7
+; GFX9-O0-NEXT: v_mov_b32_e32 v14, s6
+; GFX9-O0-NEXT: v_mov_b32_e32 v12, 0
+; GFX9-O0-NEXT: v_mov_b32_e32 v13, 0
; GFX9-O0-NEXT: v_writelane_b32 v31, s4, 8
; GFX9-O0-NEXT: v_writelane_b32 v31, s5, 9
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
@@ -1130,10 +1131,10 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v2
; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[4:5], v[0:1], s[6:7]
-; GFX9-O0-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-O0-NEXT: v_mov_b32_e32 v1, s7
; GFX9-O0-NEXT: v_mov_b32_e32 v2, s6
; GFX9-O0-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-O0-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, 0
; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:156 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:160 ; 4-byte Folded Spill
@@ -1562,11 +1563,11 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_addc_co_u32_e32 v27, vcc, -1, v5, vcc
; GFX9-NEXT: v_addc_co_u32_e32 v28, vcc, -1, v6, vcc
; GFX9-NEXT: v_addc_co_u32_e32 v29, vcc, -1, v7, vcc
-; GFX9-NEXT: v_mov_b32_e32 v16, 0
-; GFX9-NEXT: v_mov_b32_e32 v17, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: v_mov_b32_e32 v20, 0
; GFX9-NEXT: v_mov_b32_e32 v21, 0
+; GFX9-NEXT: v_mov_b32_e32 v16, 0
+; GFX9-NEXT: v_mov_b32_e32 v17, 0
; GFX9-NEXT: v_mov_b32_e32 v13, 0
; GFX9-NEXT: .LBB1_3: ; %udiv-do-while
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -2245,12 +2246,12 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:316 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:320 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:324 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v16, off, s[0:3], s32 offset:60 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v17, off, s[0:3], s32 offset:64 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:52 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:56 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:48 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v18, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v19, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_waitcnt vmcnt(9)
@@ -2260,7 +2261,7 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-O0-NEXT: s_mov_b32 s4, 64
; GFX9-O0-NEXT: v_sub_u32_e64 v20, s4, v4
-; GFX9-O0-NEXT: v_lshlrev_b64 v[20:21], v20, v[16:17]
+; GFX9-O0-NEXT: v_lshlrev_b64 v[20:21], v20, v[14:15]
; GFX9-O0-NEXT: v_mov_b32_e32 v22, v21
; GFX9-O0-NEXT: v_or_b32_e64 v5, v5, v22
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
@@ -2271,7 +2272,7 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v22, v7
; GFX9-O0-NEXT: v_cmp_lt_u32_e64 s[6:7], v4, s4
; GFX9-O0-NEXT: v_sub_u32_e64 v5, v4, s4
-; GFX9-O0-NEXT: v_lshrrev_b64 v[20:21], v5, v[16:17]
+; GFX9-O0-NEXT: v_lshrrev_b64 v[20:21], v5, v[14:15]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v21
; GFX9-O0-NEXT: v_cndmask_b32_e64 v5, v5, v22, s[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v6
@@ -2289,46 +2290,47 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[4:5]
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5
-; GFX9-O0-NEXT: v_lshrrev_b64 v[4:5], v4, v[16:17]
-; GFX9-O0-NEXT: v_mov_b32_e32 v17, v5
+; GFX9-O0-NEXT: v_lshrrev_b64 v[4:5], v4, v[14:15]
+; GFX9-O0-NEXT: v_mov_b32_e32 v15, v5
; GFX9-O0-NEXT: s_mov_b64 s[4:5], 0
; GFX9-O0-NEXT: s_mov_b32 s8, s5
-; GFX9-O0-NEXT: v_mov_b32_e32 v16, s8
-; GFX9-O0-NEXT: v_cndmask_b32_e64 v16, v16, v17, s[6:7]
+; GFX9-O0-NEXT: v_mov_b32_e32 v14, s8
+; GFX9-O0-NEXT: v_cndmask_b32_e64 v14, v14, v15, s[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v4
; GFX9-O0-NEXT: s_mov_b32 s8, s4
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s8
; GFX9-O0-NEXT: v_cndmask_b32_e64 v4, v4, v5, s[6:7]
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v5, v16
+; GFX9-O0-NEXT: v_mov_b32_e32 v5, v14
; GFX9-O0-NEXT: s_mov_b64 s[10:11], -1
; GFX9-O0-NEXT: s_mov_b32 s9, s10
-; GFX9-O0-NEXT: v_mov_b32_e32 v16, v14
-; GFX9-O0-NEXT: v_add_co_u32_e64 v18, s[6:7], v16, s9
-; GFX9-O0-NEXT: s_mov_b32 s8, s11
-; GFX9-O0-NEXT: v_mov_b32_e32 v14, v15
-; GFX9-O0-NEXT: v_mov_b32_e32 v15, s8
-; GFX9-O0-NEXT: v_addc_co_u32_e64 v14, s[6:7], v14, v15, s[6:7]
-; GFX9-O0-NEXT: ; kill: def $vgpr18 killed $vgpr18 def $vgpr18_vgpr19 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v19, v14
; GFX9-O0-NEXT: v_mov_b32_e32 v14, v12
-; GFX9-O0-NEXT: v_mov_b32_e32 v15, s9
-; GFX9-O0-NEXT: v_addc_co_u32_e64 v16, s[6:7], v14, v15, s[6:7]
+; GFX9-O0-NEXT: v_add_co_u32_e64 v14, s[6:7], v14, s9
+; GFX9-O0-NEXT: s_mov_b32 s8, s11
; GFX9-O0-NEXT: v_mov_b32_e32 v12, v13
; GFX9-O0-NEXT: v_mov_b32_e32 v13, s8
; GFX9-O0-NEXT: v_addc_co_u32_e64 v12, s[6:7], v12, v13, s[6:7]
-; GFX9-O0-NEXT: ; kill: def $vgpr16 killed $vgpr16 def $vgpr16_vgpr17 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v17, v12
-; GFX9-O0-NEXT: v_mov_b32_e32 v13, s5
-; GFX9-O0-NEXT: v_mov_b32_e32 v12, s4
-; GFX9-O0-NEXT: v_mov_b32_e32 v15, s5
-; GFX9-O0-NEXT: v_mov_b32_e32 v14, s4
-; GFX9-O0-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: ; kill: def $vgpr14 killed $vgpr14 def $vgpr14_vgpr15 killed $exec
+; GFX9-O0-NEXT: v_mov_b32_e32 v15, v12
+; GFX9-O0-NEXT: v_mov_b32_e32 v12, v16
+; GFX9-O0-NEXT: v_mov_b32_e32 v13, s9
+; GFX9-O0-NEXT: v_addc_co_u32_e64 v12, s[6:7], v12, v13, s[6:7]
+; GFX9-O0-NEXT: v_mov_b32_e32 v13, v17
+; GFX9-O0-NEXT: v_mov_b32_e32 v16, s8
+; GFX9-O0-NEXT: v_addc_co_u32_e64 v16, s[6:7], v13, v16, s[6:7]
+; GFX9-O0-NEXT: ; kill: def $vgpr12 killed $vgpr12 def $vgpr12_vgpr13 killed $exec
+; GFX9-O0-NEXT: v_mov_b32_e32 v13, v16
+; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:288 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
-; GFX9-O0-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:280 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
-; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:284 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: v_mov_b32_e32 v15, s7
+; GFX9-O0-NEXT: v_mov_b32_e32 v14, s6
+; GFX9-O0-NEXT: v_mov_b32_e32 v12, 0
+; GFX9-O0-NEXT: v_mov_b32_e32 v13, 0
; GFX9-O0-NEXT: v_writelane_b32 v31, s4, 8
; GFX9-O0-NEXT: v_writelane_b32 v31, s5, 9
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
@@ -2469,10 +2471,10 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v2
; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[4:5], v[0:1], s[6:7]
-; GFX9-O0-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-O0-NEXT: v_mov_b32_e32 v1, s7
; GFX9-O0-NEXT: v_mov_b32_e32 v2, s6
; GFX9-O0-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-O0-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, 0
; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:128 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:132 ; 4-byte Folded Spill
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-spill-cost-reset.ll b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-spill-cost-reset.ll
index d115f9bd028f2..4e8540aedcadb 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-spill-cost-reset.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-spill-cost-reset.ll
@@ -68,474 +68,474 @@ define amdgpu_kernel void @test_spill_cost_reset(
; CHECK-NEXT: liveins: $sgpr4_sgpr5
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
- ; CHECK-NEXT: early-clobber %1891:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[COPY]](p4), 16, 0 :: (dereferenceable invariant load (s256) from %ir.a0.kernarg.offset, align 16, addrspace 4)
- ; CHECK-NEXT: early-clobber %1892:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[COPY]](p4), 48, 0 :: (dereferenceable invariant load (s256) from %ir.b0.kernarg.offset, align 16, addrspace 4)
+ ; CHECK-NEXT: early-clobber %1890:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[COPY]](p4), 16, 0 :: (dereferenceable invariant load (s256) from %ir.a0.kernarg.offset, align 16, addrspace 4)
+ ; CHECK-NEXT: early-clobber %1891:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[COPY]](p4), 48, 0 :: (dereferenceable invariant load (s256) from %ir.b0.kernarg.offset, align 16, addrspace 4)
; CHECK-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 80, 0 :: (dereferenceable invariant load (s32) from %ir.n.kernarg.offset, align 16, addrspace 4)
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_1:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_2:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_3:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_4:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_5:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_6:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_7:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_8:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_9:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_10:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_11:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_12:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_128_align2 = COPY %1891.sub0_sub1_sub2_sub3
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_128_align2 = COPY %1892.sub0_sub1_sub2_sub3
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:av_128_align2 = COPY %1891.sub4_sub5_sub6_sub7
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:av_128_align2 = COPY %1892.sub4_sub5_sub6_sub7
- ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY5:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY5:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY6:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY6:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY7:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY7:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY8:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY8:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY9:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY9:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY10:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY10:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY11:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY11:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY12:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY12:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY13:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY13:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY14:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY14:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY15:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY15:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY16:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY16:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY17:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY17:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY18:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY18:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY19:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY19:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY20:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY20:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY21:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY21:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY22:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY22:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY23:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY23:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY24:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY24:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY25:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY25:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY26:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY26:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY27:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY27:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY28:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY28:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY29:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY29:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY30:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY30:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY31:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY31:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY32:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY32:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY33:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY33:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY34:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY34:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY35:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY35:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY36:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY36:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY37:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY37:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY38:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY38:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY39:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY39:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY40:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY40:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY41:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY41:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY42:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY42:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY43:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY43:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY44:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY44:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY45:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY45:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY46:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY46:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY47:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY47:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY48:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY48:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY49:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY49:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY50:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY50:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY51:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY51:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY52:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY52:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY53:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY53:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY54:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY54:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY55:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY55:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY56:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY56:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY57:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY57:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY58:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY58:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY59:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY59:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY60:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY60:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY61:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY61:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY62:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY62:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY63:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY63:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY64:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY64:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY65:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY65:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY66:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY66:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY67:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY67:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY68:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY68:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY69:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY69:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY70:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY70:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY71:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY71:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY72:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY72:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY73:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY73:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY74:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY74:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY75:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY75:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY76:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY76:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY77:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY77:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY78:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY78:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY79:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY79:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY80:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY80:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY81:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY81:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY82:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY82:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY83:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY83:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY84:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY84:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY85:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY85:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY86:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY86:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY87:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY87:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY88:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY88:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY89:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY89:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY90:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY90:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY91:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY91:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY92:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY92:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY93:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY93:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY94:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY94:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY95:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY95:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY96:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY96:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY97:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY97:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY98:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY98:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY99:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY99:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY100:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY100:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY101:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY101:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY102:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY102:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY103:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY103:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY104:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY104:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY105:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY105:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY106:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY106:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY107:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY107:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY108:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY108:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY109:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY109:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY110:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY110:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY111:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY111:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY112:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY112:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY113:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY113:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY114:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY114:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY115:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY115:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY116:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY116:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY117:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY117:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY118:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY118:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY119:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY119:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY120:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY120:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY121:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY121:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY122:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY122:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY123:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY123:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY124:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY124:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY125:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY125:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY126:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY126:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY127:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY127:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY128:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY128:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY129:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY129:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY130:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY130:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY131:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY131:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_1:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_2:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_3:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_4:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_5:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_6:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_7:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_8:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_9:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_10:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_11:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_12:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_13:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_13:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_14:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_14:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_15:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_15:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_16:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_16:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_17:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_17:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_18:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_18:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_19:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_19:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_20:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_20:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_21:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_21:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_22:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_22:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_23:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_23:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_24:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_24:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_25:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_25:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_26:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_26:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_27:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_27:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_28:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_28:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_29:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_29:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_30:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_30:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_31:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_31:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_32:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_32:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_33:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_33:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_34:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_34:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_35:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_35:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_36:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_36:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_37:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_37:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_38:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_38:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_39:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_39:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_40:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_40:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_41:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_41:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_42:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_42:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_43:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_43:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_44:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_44:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_45:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_45:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_46:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_46:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_47:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_47:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_48:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_48:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_49:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_49:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_50:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_50:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_51:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_51:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_52:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_52:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_53:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_53:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_54:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_54:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_55:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_55:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_56:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_56:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_57:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_57:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_58:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_58:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_59:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_59:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_60:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_60:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_61:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_61:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_62:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_62:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_63:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_63:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_64:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_64:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_65:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_65:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_66:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_66:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_67:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_67:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_68:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_68:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_69:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_69:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_70:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_70:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_71:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_71:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_72:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_72:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_73:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_73:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_74:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_74:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_75:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_75:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_76:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_76:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_77:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_77:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_78:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_78:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_79:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_79:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_80:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_80:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_81:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_81:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_82:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_82:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_83:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_83:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_84:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_84:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_85:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_85:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_86:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_86:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_87:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_87:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_88:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_88:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_89:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_89:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_90:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_90:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_91:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_91:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_92:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_92:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_93:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_93:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_94:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_94:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_95:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_95:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_96:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_96:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_97:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_97:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_98:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_98:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_99:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_99:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_100:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_100:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_101:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_101:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_102:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_102:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_103:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_103:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_104:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_104:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_105:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_105:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_106:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_106:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_107:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_107:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_108:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_108:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_109:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_109:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_110:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_110:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_111:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_111:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_112:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_112:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_113:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_113:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_114:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_114:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_115:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_115:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_116:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_116:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_117:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_117:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_118:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_118:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_119:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_119:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_120:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_120:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_121:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_121:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_122:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_122:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_123:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_123:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_124:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_124:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_125:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_125:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_126:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_126:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_127:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_127:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_128:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_128:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_128:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_128:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_129:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_129:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_129:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_129:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_130:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_130:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_130:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_130:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_131:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_131:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_131:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_131:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_132:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_132:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_132:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_132:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_133:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_133:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_133:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_133:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_134:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_134:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_134:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_134:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_135:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_135:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_135:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_135:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_136:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_136:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_136:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_136:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_137:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_137:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_137:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_137:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_138:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_138:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_138:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_138:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_139:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_139:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_139:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_139:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_128_align2 = COPY %1890.sub0_sub1_sub2_sub3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_128_align2 = COPY %1891.sub0_sub1_sub2_sub3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:av_128_align2 = COPY %1890.sub4_sub5_sub6_sub7
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:av_128_align2 = COPY %1891.sub4_sub5_sub6_sub7
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.loop:
; CHECK-NEXT: successors: %bb.2(0x04000000), %bb.1(0x7c000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_12]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY131:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY131]], 8, [[COPY131]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY130:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY130]], 8, [[COPY130]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY129:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY129]], 8, [[COPY129]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY128:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY128]], 8, [[COPY128]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY127:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY127]], 8, [[COPY127]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_11]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY126:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY126]], 8, [[COPY126]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY125:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY125]], 8, [[COPY125]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY124:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY124]], 8, [[COPY124]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_10]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY123:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY123]], 8, [[COPY123]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY122:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY122]], 8, [[COPY122]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY121:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY121]], 8, [[COPY121]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_9]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY120:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY120]], 8, [[COPY120]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY119:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY119]], 8, [[COPY119]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY118:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY118]], 8, [[COPY118]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_8]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY117:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY117]], 8, [[COPY117]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY116:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY116]], 8, [[COPY116]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY115:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY115]], 8, [[COPY115]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_7]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY114:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY114]], 8, [[COPY114]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY113:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY113]], 8, [[COPY113]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY112:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY112]], 8, [[COPY112]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_6]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY111:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY111]], 8, [[COPY111]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY110:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY110]], 8, [[COPY110]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY109:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY109]], 8, [[COPY109]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_5]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY108:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY108]], 8, [[COPY108]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY107:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY107]], 8, [[COPY107]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY106:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY106]], 8, [[COPY106]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_4]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY105:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY105]], 8, [[COPY105]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY104:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY104]], 8, [[COPY104]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY103:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY103]], 8, [[COPY103]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_3]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY102:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY102]], 8, [[COPY102]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY101:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY101]], 8, [[COPY101]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY100:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY100]], 8, [[COPY100]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_2]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY99:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY99]], 8, [[COPY99]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY98:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY98]], 8, [[COPY98]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY97:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY97]], 8, [[COPY97]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_1]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY96:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY96]], 8, [[COPY96]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY95:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY95]], 8, [[COPY95]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY94:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY94]], 8, [[COPY94]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY93:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY93]], 8, [[COPY93]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY92:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY92]], 8, [[COPY92]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY91:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY91]], 8, [[COPY91]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY3]], [[COPY4]], [[V_MOV_B32_e32_12]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_139:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_139]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_]], 8, [[V_MOV_B32_e32_]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_1]], 8, [[V_MOV_B32_e32_1]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_2]], 8, [[V_MOV_B32_e32_2]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_3]], 8, [[V_MOV_B32_e32_3]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_4]], 8, [[V_MOV_B32_e32_4]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_138:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_138]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_5]], 8, [[V_MOV_B32_e32_5]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_6]], 8, [[V_MOV_B32_e32_6]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_7]], 8, [[V_MOV_B32_e32_7]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_137:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_137]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_8]], 8, [[V_MOV_B32_e32_8]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_9]], 8, [[V_MOV_B32_e32_9]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_10]], 8, [[V_MOV_B32_e32_10]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_136:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_136]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_11]], 8, [[V_MOV_B32_e32_11]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_12]], 8, [[V_MOV_B32_e32_12]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_13:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_13]], 8, [[V_MOV_B32_e32_13]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_135:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_135]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_14:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_14]], 8, [[V_MOV_B32_e32_14]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_15:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_15]], 8, [[V_MOV_B32_e32_15]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_16:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_16]], 8, [[V_MOV_B32_e32_16]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_134:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_134]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_17:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_17]], 8, [[V_MOV_B32_e32_17]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_18:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_18]], 8, [[V_MOV_B32_e32_18]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_19:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_19]], 8, [[V_MOV_B32_e32_19]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_133:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_133]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_20:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_20]], 8, [[V_MOV_B32_e32_20]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_21:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_21]], 8, [[V_MOV_B32_e32_21]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_22:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_22]], 8, [[V_MOV_B32_e32_22]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_132:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_132]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_23:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_23]], 8, [[V_MOV_B32_e32_23]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_24:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_24]], 8, [[V_MOV_B32_e32_24]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_25:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_25]], 8, [[V_MOV_B32_e32_25]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_131:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_131]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_26:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_26]], 8, [[V_MOV_B32_e32_26]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_27:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_27]], 8, [[V_MOV_B32_e32_27]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_28:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_28]], 8, [[V_MOV_B32_e32_28]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_130:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_130]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_29:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_29]], 8, [[V_MOV_B32_e32_29]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_30:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_30]], 8, [[V_MOV_B32_e32_30]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_31:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_31]], 8, [[V_MOV_B32_e32_31]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_129:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_129]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_32:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_32]], 8, [[V_MOV_B32_e32_32]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_33:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_33]], 8, [[V_MOV_B32_e32_33]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_34:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_34]], 8, [[V_MOV_B32_e32_34]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_128:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_128]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_35:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_35]], 8, [[V_MOV_B32_e32_35]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_36:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_36]], 8, [[V_MOV_B32_e32_36]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_37:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_37]], 8, [[V_MOV_B32_e32_37]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_38:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_38]], 8, [[V_MOV_B32_e32_38]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_39:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_39]], 8, [[V_MOV_B32_e32_39]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_40:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_40]], 8, [[V_MOV_B32_e32_40]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY3]], [[COPY4]], [[V_MOV_B32_e32_139]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_1:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY90:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY90]], 8, [[COPY90]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY89:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY89]], 8, [[COPY89]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY88:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY88]], 8, [[COPY88]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_41:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_41]], 8, [[V_MOV_B32_e32_41]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_42:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_42]], 8, [[V_MOV_B32_e32_42]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_43:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_43]], 8, [[V_MOV_B32_e32_43]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_2:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY3]], [[COPY4]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY87:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY87]], 8, [[COPY87]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY86:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY86]], 8, [[COPY86]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY85:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY85]], 8, [[COPY85]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY84:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY84]], 8, [[COPY84]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY83:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY83]], 8, [[COPY83]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY82:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY82]], 8, [[COPY82]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY81:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY81]], 8, [[COPY81]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY80:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY80]], 8, [[COPY80]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY79:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY79]], 8, [[COPY79]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY78:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY78]], 8, [[COPY78]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY77:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY77]], 8, [[COPY77]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY76:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY76]], 8, [[COPY76]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY75:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY75]], 8, [[COPY75]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY74:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY74]], 8, [[COPY74]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY73:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY73]], 8, [[COPY73]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY72:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY72]], 8, [[COPY72]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY71:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY71]], 8, [[COPY71]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY70:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY70]], 8, [[COPY70]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY69:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY69]], 8, [[COPY69]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY68:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY68]], 8, [[COPY68]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY67:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY67]], 8, [[COPY67]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY66:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY66]], 8, [[COPY66]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY65:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY65]], 8, [[COPY65]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY64:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY64]], 8, [[COPY64]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY63:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY63]], 8, [[COPY63]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY62:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY62]], 8, [[COPY62]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY61:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY61]], 8, [[COPY61]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY60:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY60]], 8, [[COPY60]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY59:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY59]], 8, [[COPY59]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY58:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY58]], 8, [[COPY58]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY57:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY57]], 8, [[COPY57]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY56:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY56]], 8, [[COPY56]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY55:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY55]], 8, [[COPY55]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY54:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY54]], 8, [[COPY54]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY53:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY53]], 8, [[COPY53]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY52:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY52]], 8, [[COPY52]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY51:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY51]], 8, [[COPY51]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY50:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY50]], 8, [[COPY50]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY49:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY49]], 8, [[COPY49]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY48:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY48]], 8, [[COPY48]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY47:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY47]], 8, [[COPY47]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY46:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY46]], 8, [[COPY46]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY45:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY45]], 8, [[COPY45]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY44:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY44]], 8, [[COPY44]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY43:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY43]], 8, [[COPY43]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY42:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY42]], 8, [[COPY42]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY41:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY41]], 8, [[COPY41]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY40:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY40]], 8, [[COPY40]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY39:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY39]], 8, [[COPY39]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY38:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY38]], 8, [[COPY38]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY37:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY37]], 8, [[COPY37]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY36:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY36]], 8, [[COPY36]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY35:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY35]], 8, [[COPY35]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY34:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY34]], 8, [[COPY34]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY33:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY33]], 8, [[COPY33]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY32:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY32]], 8, [[COPY32]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY31:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY31]], 8, [[COPY31]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY30:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY30]], 8, [[COPY30]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY29:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY29]], 8, [[COPY29]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY28:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY28]], 8, [[COPY28]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY27:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY27]], 8, [[COPY27]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY26:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY26]], 8, [[COPY26]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY25:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY25]], 8, [[COPY25]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY24:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY24]], 8, [[COPY24]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY23:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY23]], 8, [[COPY23]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY22:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY22]], 8, [[COPY22]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY21:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY21]], 8, [[COPY21]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY20:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY20]], 8, [[COPY20]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY19:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY19]], 8, [[COPY19]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY18:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY18]], 8, [[COPY18]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY17:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY17]], 8, [[COPY17]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY16:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY16]], 8, [[COPY16]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY15]], 8, [[COPY15]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY14]], 8, [[COPY14]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY13]], 8, [[COPY13]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY12]], 8, [[COPY12]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY11]], 8, [[COPY11]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY10]], 8, [[COPY10]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY9]], 8, [[COPY9]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY8]], 8, [[COPY8]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY7]], 8, [[COPY7]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY6]], 8, [[COPY6]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY5]], 8, [[COPY5]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_44:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_44]], 8, [[V_MOV_B32_e32_44]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_45:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_45]], 8, [[V_MOV_B32_e32_45]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_46:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_46]], 8, [[V_MOV_B32_e32_46]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_47:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_47]], 8, [[V_MOV_B32_e32_47]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_48:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_48]], 8, [[V_MOV_B32_e32_48]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_49:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_49]], 8, [[V_MOV_B32_e32_49]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_50:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_50]], 8, [[V_MOV_B32_e32_50]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_51:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_51]], 8, [[V_MOV_B32_e32_51]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_52:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_52]], 8, [[V_MOV_B32_e32_52]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_53:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_53]], 8, [[V_MOV_B32_e32_53]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_54:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_54]], 8, [[V_MOV_B32_e32_54]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_55:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_55]], 8, [[V_MOV_B32_e32_55]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_56:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_56]], 8, [[V_MOV_B32_e32_56]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_57:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_57]], 8, [[V_MOV_B32_e32_57]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_58:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_58]], 8, [[V_MOV_B32_e32_58]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_59:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_59]], 8, [[V_MOV_B32_e32_59]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_60:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_60]], 8, [[V_MOV_B32_e32_60]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_61:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_61]], 8, [[V_MOV_B32_e32_61]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_62:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_62]], 8, [[V_MOV_B32_e32_62]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_63:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_63]], 8, [[V_MOV_B32_e32_63]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_64:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_64]], 8, [[V_MOV_B32_e32_64]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_65:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_65]], 8, [[V_MOV_B32_e32_65]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_66:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_66]], 8, [[V_MOV_B32_e32_66]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_67:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_67]], 8, [[V_MOV_B32_e32_67]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_68:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_68]], 8, [[V_MOV_B32_e32_68]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_69:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_69]], 8, [[V_MOV_B32_e32_69]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_70:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_70]], 8, [[V_MOV_B32_e32_70]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_71:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_71]], 8, [[V_MOV_B32_e32_71]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_72:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_72]], 8, [[V_MOV_B32_e32_72]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_73:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_73]], 8, [[V_MOV_B32_e32_73]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_74:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_74]], 8, [[V_MOV_B32_e32_74]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_75:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_75]], 8, [[V_MOV_B32_e32_75]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_76:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_76]], 8, [[V_MOV_B32_e32_76]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_77:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_77]], 8, [[V_MOV_B32_e32_77]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_78:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_78]], 8, [[V_MOV_B32_e32_78]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_79:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_79]], 8, [[V_MOV_B32_e32_79]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_80:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_80]], 8, [[V_MOV_B32_e32_80]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_81:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_81]], 8, [[V_MOV_B32_e32_81]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_82:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_82]], 8, [[V_MOV_B32_e32_82]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_83:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_83]], 8, [[V_MOV_B32_e32_83]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_84:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_84]], 8, [[V_MOV_B32_e32_84]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_85:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_85]], 8, [[V_MOV_B32_e32_85]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_86:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_86]], 8, [[V_MOV_B32_e32_86]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_87:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_87]], 8, [[V_MOV_B32_e32_87]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_88:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_88]], 8, [[V_MOV_B32_e32_88]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_89:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_89]], 8, [[V_MOV_B32_e32_89]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_90:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_90]], 8, [[V_MOV_B32_e32_90]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_91:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_91]], 8, [[V_MOV_B32_e32_91]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_92:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_92]], 8, [[V_MOV_B32_e32_92]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_93:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_93]], 8, [[V_MOV_B32_e32_93]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_94:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_94]], 8, [[V_MOV_B32_e32_94]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_95:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_95]], 8, [[V_MOV_B32_e32_95]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_96:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_96]], 8, [[V_MOV_B32_e32_96]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_97:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_97]], 8, [[V_MOV_B32_e32_97]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_98:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_98]], 8, [[V_MOV_B32_e32_98]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_99:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_99]], 8, [[V_MOV_B32_e32_99]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_100:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_100]], 8, [[V_MOV_B32_e32_100]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_101:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_101]], 8, [[V_MOV_B32_e32_101]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_102:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_102]], 8, [[V_MOV_B32_e32_102]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_103:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_103]], 8, [[V_MOV_B32_e32_103]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_104:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_104]], 8, [[V_MOV_B32_e32_104]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_105:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_105]], 8, [[V_MOV_B32_e32_105]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_106:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_106]], 8, [[V_MOV_B32_e32_106]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_107:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_107]], 8, [[V_MOV_B32_e32_107]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_108:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_108]], 8, [[V_MOV_B32_e32_108]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_109:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_109]], 8, [[V_MOV_B32_e32_109]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_110:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_110]], 8, [[V_MOV_B32_e32_110]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_111:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_111]], 8, [[V_MOV_B32_e32_111]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_112:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_112]], 8, [[V_MOV_B32_e32_112]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_113:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_113]], 8, [[V_MOV_B32_e32_113]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_114:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_114]], 8, [[V_MOV_B32_e32_114]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_115:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_115]], 8, [[V_MOV_B32_e32_115]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_116:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_116]], 8, [[V_MOV_B32_e32_116]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_117:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_117]], 8, [[V_MOV_B32_e32_117]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_118:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_118]], 8, [[V_MOV_B32_e32_118]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_119:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_119]], 8, [[V_MOV_B32_e32_119]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_120:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_120]], 8, [[V_MOV_B32_e32_120]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_121:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_121]], 8, [[V_MOV_B32_e32_121]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_122:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_122]], 8, [[V_MOV_B32_e32_122]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_123:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_123]], 8, [[V_MOV_B32_e32_123]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_124:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_124]], 8, [[V_MOV_B32_e32_124]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_125:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_125]], 8, [[V_MOV_B32_e32_125]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_126:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_126]], 8, [[V_MOV_B32_e32_126]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_MOV_B32_]], 1, implicit-def dead $scc
; CHECK-NEXT: S_CMP_LT_I32 [[S_MOV_B32_]], [[S_LOAD_DWORD_IMM]], implicit-def $scc
- ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_]], 8, [[V_MOV_B32_e32_]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_127:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_127]], 8, [[V_MOV_B32_e32_127]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit killed $scc
; CHECK-NEXT: S_BRANCH %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.epilogue:
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 0, 0 :: (dereferenceable invariant load (s64) from %ir.out.kernarg.offset550, align 16, addrspace 4)
- ; CHECK-NEXT: [[V_MOV_B32_e32_13:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR [[V_MOV_B32_e32_13]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_2]].sub0, [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s32) into %ir.out.load, addrspace 1)
+ ; CHECK-NEXT: [[V_MOV_B32_e32_140:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR [[V_MOV_B32_e32_140]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_2]].sub0, [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s32) into %ir.out.load, addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
ptr addrspace(1) %out,
<8 x half> %a0, <8 x half> %a1,
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-reserved-wwm-vgpr.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-reserved-wwm-vgpr.ll
index 3128b69788b32..d0e4a98ca3d57 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-reserved-wwm-vgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-reserved-wwm-vgpr.ll
@@ -20,8 +20,8 @@ define amdgpu_kernel void @reserved_wwm_vgpr_not_in_alloc_order(i32 %arg6, i1 %a
; CHECK-NEXT: s_load_dword s8, s[4:5], 0x10
; CHECK-NEXT: s_load_dwordx4 s[12:15], s[4:5], 0x20
; CHECK-NEXT: ; implicit-def: $vgpr31 : SGPR spill to VGPR lane
-; CHECK-NEXT: v_mov_b32_e32 v30, 0
-; CHECK-NEXT: v_mov_b32_e32 v4, v30
+; CHECK-NEXT: v_mov_b32_e32 v0, 0
+; CHECK-NEXT: v_mov_b32_e32 v1, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_bitcmp1_b32 s1, 0
; CHECK-NEXT: s_cselect_b64 s[6:7], -1, 0
@@ -30,13 +30,13 @@ define amdgpu_kernel void @reserved_wwm_vgpr_not_in_alloc_order(i32 %arg6, i1 %a
; CHECK-NEXT: s_bitcmp1_b32 s1, 8
; CHECK-NEXT: v_writelane_b32 v31, s13, 1
; CHECK-NEXT: v_writelane_b32 v31, s14, 2
+; CHECK-NEXT: v_writelane_b32 v31, s15, 3
; CHECK-NEXT: s_cselect_b64 s[16:17], -1, 0
; CHECK-NEXT: s_bitcmp1_b32 s1, 16
-; CHECK-NEXT: v_writelane_b32 v31, s15, 3
-; CHECK-NEXT: s_cselect_b64 vcc, -1, 0
-; CHECK-NEXT: s_bitcmp1_b32 s8, 0
; CHECK-NEXT: s_load_dwordx4 s[12:15], s[4:5], 0x40
; CHECK-NEXT: s_load_dword s24, s[4:5], 0x90
+; CHECK-NEXT: s_cselect_b64 vcc, -1, 0
+; CHECK-NEXT: s_bitcmp1_b32 s8, 0
; CHECK-NEXT: s_cselect_b64 s[34:35], -1, 0
; CHECK-NEXT: s_bitcmp1_b32 s8, 8
; CHECK-NEXT: s_load_dword s1, s[4:5], 0x50
@@ -46,37 +46,33 @@ define amdgpu_kernel void @reserved_wwm_vgpr_not_in_alloc_order(i32 %arg6, i1 %a
; CHECK-NEXT: s_cselect_b64 s[36:37], -1, 0
; CHECK-NEXT: s_bitcmp1_b32 s9, 0
; CHECK-NEXT: v_writelane_b32 v31, s13, 5
+; CHECK-NEXT: v_writelane_b32 v31, s14, 6
; CHECK-NEXT: s_cselect_b64 s[38:39], -1, 0
; CHECK-NEXT: s_bitcmp1_b32 s8, 0
-; CHECK-NEXT: v_writelane_b32 v31, s14, 6
+; CHECK-NEXT: v_writelane_b32 v31, s15, 7
; CHECK-NEXT: s_cselect_b64 s[40:41], -1, 0
+; CHECK-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x60
; CHECK-NEXT: s_bitcmp1_b32 s24, 0
; CHECK-NEXT: s_mov_b32 s24, 0
-; CHECK-NEXT: v_writelane_b32 v31, s15, 7
-; CHECK-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x60
+; CHECK-NEXT: v_mov_b32_e32 v2, 0
+; CHECK-NEXT: v_mov_b32_e32 v3, 0
+; CHECK-NEXT: v_mov_b32_e32 v4, 0
+; CHECK-NEXT: v_mov_b32_e32 v5, 0
+; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
+; CHECK-NEXT: v_mov_b32_e32 v8, 0
+; CHECK-NEXT: v_mov_b32_e32 v9, 0
+; CHECK-NEXT: v_mov_b32_e32 v10, 0
+; CHECK-NEXT: v_mov_b32_e32 v11, 0
+; CHECK-NEXT: v_mov_b32_e32 v12, 0
+; CHECK-NEXT: v_mov_b32_e32 v13, 0
+; CHECK-NEXT: v_mov_b32_e32 v14, 0
+; CHECK-NEXT: v_mov_b32_e32 v15, 0
; CHECK-NEXT: s_mov_b32 s25, s24
; CHECK-NEXT: s_mov_b32 s26, s24
; CHECK-NEXT: s_mov_b32 s27, s24
-; CHECK-NEXT: v_mov_b64_e32 v[0:1], s[24:25]
-; CHECK-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
; CHECK-NEXT: s_cselect_b64 s[42:43], -1, 0
; CHECK-NEXT: s_lshl_b32 s60, s1, 1
-; CHECK-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:64 ; 16-byte Folded Spill
-; CHECK-NEXT: v_mov_b32_e32 v5, v30
-; CHECK-NEXT: v_mov_b32_e32 v6, v30
-; CHECK-NEXT: v_mov_b32_e32 v0, 0
-; CHECK-NEXT: v_mov_b32_e32 v1, v30
-; CHECK-NEXT: v_mov_b32_e32 v2, v30
-; CHECK-NEXT: v_mov_b32_e32 v3, v30
-; CHECK-NEXT: v_mov_b32_e32 v7, v30
-; CHECK-NEXT: v_mov_b32_e32 v8, v30
-; CHECK-NEXT: v_mov_b32_e32 v9, v30
-; CHECK-NEXT: v_mov_b32_e32 v10, v30
-; CHECK-NEXT: v_mov_b32_e32 v11, v30
-; CHECK-NEXT: v_mov_b32_e32 v12, v30
-; CHECK-NEXT: v_mov_b32_e32 v13, v30
-; CHECK-NEXT: v_mov_b32_e32 v14, v30
-; CHECK-NEXT: v_mov_b32_e32 v15, v30
; CHECK-NEXT: v_accvgpr_write_b32 a31, v15
; CHECK-NEXT: v_accvgpr_write_b32 a30, v14
; CHECK-NEXT: v_accvgpr_write_b32 a29, v13
@@ -93,11 +89,34 @@ define amdgpu_kernel void @reserved_wwm_vgpr_not_in_alloc_order(i32 %arg6, i1 %a
; CHECK-NEXT: v_accvgpr_write_b32 a18, v2
; CHECK-NEXT: v_accvgpr_write_b32 a17, v1
; CHECK-NEXT: v_accvgpr_write_b32 a16, v0
-; CHECK-NEXT: v_mov_b32_e32 v0, s60
+; CHECK-NEXT: v_mov_b64_e32 v[0:1], s[24:25]
+; CHECK-NEXT: v_mov_b64_e32 v[2:3], s[26:27]
+; CHECK-NEXT: scratch_store_dwordx4 off, v[0:3], off offset:64 ; 16-byte Folded Spill
+; CHECK-NEXT: v_mov_b32_e32 v30, 0
; CHECK-NEXT: s_mov_b64 s[52:53], -1
+; CHECK-NEXT: v_mov_b32_e32 v0, s60
; CHECK-NEXT: s_mov_b64 s[20:21], s[4:5]
; CHECK-NEXT: s_mov_b64 s[4:5], s[16:17]
; CHECK-NEXT: s_mov_b64 s[54:55], 0
+; CHECK-NEXT: v_mov_b32_e32 v18, 0
+; CHECK-NEXT: v_mov_b32_e32 v17, 0
+; CHECK-NEXT: v_mov_b32_e32 v16, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a0, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a1, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a2, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a3, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a4, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a5, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a6, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a7, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a8, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a9, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a10, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a11, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a12, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a13, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a14, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a15, 0
; CHECK-NEXT: s_mov_b64 s[58:59], 0
; CHECK-NEXT: s_mov_b64 s[56:57], 0
; CHECK-NEXT: s_mov_b64 s[44:45], 0
@@ -105,26 +124,7 @@ define amdgpu_kernel void @reserved_wwm_vgpr_not_in_alloc_order(i32 %arg6, i1 %a
; CHECK-NEXT: s_mov_b32 s1, s24
; CHECK-NEXT: s_mov_b64 s[48:49], 0
; CHECK-NEXT: s_mov_b64 s[50:51], 0
-; CHECK-NEXT: v_accvgpr_write_b32 a0, 0
-; CHECK-NEXT: v_accvgpr_write_b32 a1, v30
-; CHECK-NEXT: v_accvgpr_write_b32 a2, v30
-; CHECK-NEXT: v_accvgpr_write_b32 a3, v30
-; CHECK-NEXT: v_accvgpr_write_b32 a4, v30
-; CHECK-NEXT: v_accvgpr_write_b32 a5, v30
-; CHECK-NEXT: v_accvgpr_write_b32 a6, v30
-; CHECK-NEXT: v_accvgpr_write_b32 a7, v30
-; CHECK-NEXT: v_accvgpr_write_b32 a8, v30
-; CHECK-NEXT: v_accvgpr_write_b32 a9, v30
-; CHECK-NEXT: v_accvgpr_write_b32 a10, v30
-; CHECK-NEXT: v_accvgpr_write_b32 a11, v30
-; CHECK-NEXT: v_accvgpr_write_b32 a12, v30
-; CHECK-NEXT: v_accvgpr_write_b32 a13, v30
-; CHECK-NEXT: v_accvgpr_write_b32 a14, v30
-; CHECK-NEXT: v_accvgpr_write_b32 a15, v30
; CHECK-NEXT: s_mov_b32 s33, s24
-; CHECK-NEXT: v_mov_b32_e32 v16, 0
-; CHECK-NEXT: v_mov_b32_e32 v17, v30
-; CHECK-NEXT: v_mov_b32_e32 v18, v30
; CHECK-NEXT: v_mov_b32_e32 v19, v30
; CHECK-NEXT: s_mov_b32 s62, s24
; CHECK-NEXT: s_mov_b32 s63, s24
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-multi-store-codegen.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-multi-store-codegen.ll
index 2d35135b5f757..ecdae36208bfa 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-multi-store-codegen.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-multi-store-codegen.ll
@@ -12,64 +12,64 @@ define amdgpu_kernel void @multi_store_spill_slot() #0 {
; CHECK-NEXT: v_mov_b32_e32 v3, 0
; CHECK-NEXT: s_mov_b32 s1, s0
; CHECK-NEXT: v_mov_b32_e32 v22, 0x3c003c00
+; CHECK-NEXT: v_mov_b32_e32 v178, 0
+; CHECK-NEXT: v_mov_b32_e32 v18, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a66, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a1, 0
+; CHECK-NEXT: v_mov_b32_e32 v210, 0
+; CHECK-NEXT: v_mov_b32_e32 v34, 0
+; CHECK-NEXT: v_mov_b32_e32 v50, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a16, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a115, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a114, 0
+; CHECK-NEXT: v_mov_b32_e32 v82, 0
+; CHECK-NEXT: v_mov_b32_e32 v98, 0
+; CHECK-NEXT: v_mov_b32_e32 v194, 0
+; CHECK-NEXT: v_mov_b32_e32 v243, 0
+; CHECK-NEXT: v_mov_b32_e32 v114, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a33, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a32, 0
+; CHECK-NEXT: v_mov_b32_e32 v145, 0
+; CHECK-NEXT: v_mov_b32_e32 v146, 0
+; CHECK-NEXT: v_mov_b32_e32 v147, 0
; CHECK-NEXT: v_pk_mov_b32 v[24:25], s[0:1], s[0:1] op_sel:[0,1]
; CHECK-NEXT: v_mov_b32_e32 v0, 0x7fc00000
-; CHECK-NEXT: v_accvgpr_write_b32 a1, 1.0
+; CHECK-NEXT: v_accvgpr_write_b32 a17, 1.0
; CHECK-NEXT: v_mov_b32_e32 v23, v22
; CHECK-NEXT: s_and_b64 vcc, exec, -1
; CHECK-NEXT: v_mov_b32_e32 v2, v3
-; CHECK-NEXT: v_mov_b32_e32 v178, v3
-; CHECK-NEXT: v_mov_b32_e32 v18, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a66, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a17, v3
-; CHECK-NEXT: v_mov_b32_e32 v210, v3
-; CHECK-NEXT: v_mov_b32_e32 v34, v3
-; CHECK-NEXT: v_mov_b32_e32 v50, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a0, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a115, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a114, v3
-; CHECK-NEXT: v_mov_b32_e32 v82, v3
-; CHECK-NEXT: v_mov_b32_e32 v98, v3
-; CHECK-NEXT: v_mov_b32_e32 v194, v3
-; CHECK-NEXT: v_mov_b32_e32 v243, v3
-; CHECK-NEXT: v_mov_b32_e32 v114, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a33, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a32, v3
-; CHECK-NEXT: v_mov_b32_e32 v145, v3
-; CHECK-NEXT: v_mov_b32_e32 v146, v3
-; CHECK-NEXT: v_mov_b32_e32 v147, v3
; CHECK-NEXT: .LBB0_1: ; %do.body
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_accvgpr_mov_b32 a2, a1
-; CHECK-NEXT: v_accvgpr_mov_b32 a3, a1
-; CHECK-NEXT: v_accvgpr_mov_b32 a4, a1
-; CHECK-NEXT: v_accvgpr_mov_b32 a5, a1
-; CHECK-NEXT: v_accvgpr_mov_b32 a6, a1
-; CHECK-NEXT: v_accvgpr_mov_b32 a7, a1
-; CHECK-NEXT: v_accvgpr_mov_b32 a8, a1
-; CHECK-NEXT: v_accvgpr_mov_b32 a9, a1
-; CHECK-NEXT: v_accvgpr_mov_b32 a10, a1
-; CHECK-NEXT: v_accvgpr_mov_b32 a11, a1
-; CHECK-NEXT: v_accvgpr_mov_b32 a12, a1
-; CHECK-NEXT: v_accvgpr_mov_b32 a13, a1
-; CHECK-NEXT: v_accvgpr_mov_b32 a14, a1
-; CHECK-NEXT: v_accvgpr_mov_b32 a15, a1
-; CHECK-NEXT: v_accvgpr_mov_b32 a49, a15
-; CHECK-NEXT: v_accvgpr_mov_b32 a48, a14
-; CHECK-NEXT: v_accvgpr_mov_b32 a47, a13
-; CHECK-NEXT: v_accvgpr_mov_b32 a46, a12
-; CHECK-NEXT: v_accvgpr_mov_b32 a45, a11
-; CHECK-NEXT: v_accvgpr_mov_b32 a44, a10
-; CHECK-NEXT: v_accvgpr_mov_b32 a43, a9
-; CHECK-NEXT: v_accvgpr_mov_b32 a42, a8
-; CHECK-NEXT: v_accvgpr_mov_b32 a41, a7
-; CHECK-NEXT: v_accvgpr_mov_b32 a40, a6
-; CHECK-NEXT: v_accvgpr_mov_b32 a39, a5
-; CHECK-NEXT: v_accvgpr_mov_b32 a38, a4
-; CHECK-NEXT: v_accvgpr_mov_b32 a37, a3
-; CHECK-NEXT: v_accvgpr_mov_b32 a36, a2
-; CHECK-NEXT: v_accvgpr_mov_b32 a35, a1
-; CHECK-NEXT: v_accvgpr_mov_b32 a34, a0
+; CHECK-NEXT: v_accvgpr_mov_b32 a18, a17
+; CHECK-NEXT: v_accvgpr_mov_b32 a19, a17
+; CHECK-NEXT: v_accvgpr_mov_b32 a20, a17
+; CHECK-NEXT: v_accvgpr_mov_b32 a21, a17
+; CHECK-NEXT: v_accvgpr_mov_b32 a22, a17
+; CHECK-NEXT: v_accvgpr_mov_b32 a23, a17
+; CHECK-NEXT: v_accvgpr_mov_b32 a24, a17
+; CHECK-NEXT: v_accvgpr_mov_b32 a25, a17
+; CHECK-NEXT: v_accvgpr_mov_b32 a26, a17
+; CHECK-NEXT: v_accvgpr_mov_b32 a27, a17
+; CHECK-NEXT: v_accvgpr_mov_b32 a28, a17
+; CHECK-NEXT: v_accvgpr_mov_b32 a29, a17
+; CHECK-NEXT: v_accvgpr_mov_b32 a30, a17
+; CHECK-NEXT: v_accvgpr_mov_b32 a31, a17
+; CHECK-NEXT: v_accvgpr_mov_b32 a49, a31
+; CHECK-NEXT: v_accvgpr_mov_b32 a48, a30
+; CHECK-NEXT: v_accvgpr_mov_b32 a47, a29
+; CHECK-NEXT: v_accvgpr_mov_b32 a46, a28
+; CHECK-NEXT: v_accvgpr_mov_b32 a45, a27
+; CHECK-NEXT: v_accvgpr_mov_b32 a44, a26
+; CHECK-NEXT: v_accvgpr_mov_b32 a43, a25
+; CHECK-NEXT: v_accvgpr_mov_b32 a42, a24
+; CHECK-NEXT: v_accvgpr_mov_b32 a41, a23
+; CHECK-NEXT: v_accvgpr_mov_b32 a40, a22
+; CHECK-NEXT: v_accvgpr_mov_b32 a39, a21
+; CHECK-NEXT: v_accvgpr_mov_b32 a38, a20
+; CHECK-NEXT: v_accvgpr_mov_b32 a37, a19
+; CHECK-NEXT: v_accvgpr_mov_b32 a36, a18
+; CHECK-NEXT: v_accvgpr_mov_b32 a35, a17
+; CHECK-NEXT: v_accvgpr_mov_b32 a34, a16
; CHECK-NEXT: v_accvgpr_write_b32 a116, v3
; CHECK-NEXT: v_accvgpr_write_b32 a117, v3
; CHECK-NEXT: v_mfma_f32_32x32x8f16 a[34:49], v[24:25], v[24:25], a[34:49]
@@ -163,7 +163,7 @@ define amdgpu_kernel void @multi_store_spill_slot() #0 {
; CHECK-NEXT: v_mov_b32_e32 v15, v3
; CHECK-NEXT: v_mov_b32_e32 v16, v3
; CHECK-NEXT: v_mov_b32_e32 v17, v3
-; CHECK-NEXT: v_accvgpr_mov_b32 a16, a66
+; CHECK-NEXT: v_accvgpr_mov_b32 a0, a66
; CHECK-NEXT: v_accvgpr_write_b32 a79, v17
; CHECK-NEXT: v_accvgpr_write_b32 a78, v16
; CHECK-NEXT: v_accvgpr_write_b32 a77, v15
@@ -277,21 +277,21 @@ define amdgpu_kernel void @multi_store_spill_slot() #0 {
; CHECK-NEXT: v_mov_b32_e32 v142, v3
; CHECK-NEXT: v_mov_b32_e32 v143, v3
; CHECK-NEXT: v_mov_b32_e32 v144, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a18, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a19, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a20, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a21, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a22, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a23, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a24, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a25, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a26, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a27, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a28, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a29, v3
+; CHECK-NEXT: v_accvgpr_write_b32 a2, v3
+; CHECK-NEXT: v_accvgpr_write_b32 a3, v3
+; CHECK-NEXT: v_accvgpr_write_b32 a4, v3
+; CHECK-NEXT: v_accvgpr_write_b32 a5, v3
+; CHECK-NEXT: v_accvgpr_write_b32 a6, v3
+; CHECK-NEXT: v_accvgpr_write_b32 a7, v3
+; CHECK-NEXT: v_accvgpr_write_b32 a8, v3
+; CHECK-NEXT: v_accvgpr_write_b32 a9, v3
+; CHECK-NEXT: v_accvgpr_write_b32 a10, v3
+; CHECK-NEXT: v_accvgpr_write_b32 a11, v3
+; CHECK-NEXT: v_accvgpr_write_b32 a12, v3
+; CHECK-NEXT: v_accvgpr_write_b32 a13, v3
; CHECK-NEXT: v_mfma_f32_32x32x8f16 v[210:225], v[24:25], v[24:25], v[210:225]
-; CHECK-NEXT: v_accvgpr_write_b32 a30, v3
-; CHECK-NEXT: v_accvgpr_write_b32 a31, v3
+; CHECK-NEXT: v_accvgpr_write_b32 a14, v3
+; CHECK-NEXT: v_accvgpr_write_b32 a15, v3
; CHECK-NEXT: v_mov_b32_e32 v35, v3
; CHECK-NEXT: v_mov_b32_e32 v36, v3
; CHECK-NEXT: v_mov_b32_e32 v37, v3
@@ -324,8 +324,8 @@ define amdgpu_kernel void @multi_store_spill_slot() #0 {
; CHECK-NEXT: v_mfma_f32_32x32x8f16 v[146:161], v[24:25], v[24:25], v[146:161]
; CHECK-NEXT: v_mov_b32_e32 v112, v3
; CHECK-NEXT: v_mov_b32_e32 v113, v3
-; CHECK-NEXT: v_accvgpr_mov_b32 a81, a31
-; CHECK-NEXT: v_accvgpr_mov_b32 a80, a30
+; CHECK-NEXT: v_accvgpr_mov_b32 a81, a15
+; CHECK-NEXT: v_accvgpr_mov_b32 a80, a14
; CHECK-NEXT: v_mov_b32_e32 v51, v3
; CHECK-NEXT: v_mov_b32_e32 v52, v3
; CHECK-NEXT: v_mov_b32_e32 v53, v3
@@ -357,21 +357,21 @@ define amdgpu_kernel void @multi_store_spill_slot() #0 {
; CHECK-NEXT: v_mov_b32_e32 v94, v3
; CHECK-NEXT: v_mov_b32_e32 v95, v3
; CHECK-NEXT: v_mov_b32_e32 v96, v3
-; CHECK-NEXT: v_accvgpr_mov_b32 a79, a29
-; CHECK-NEXT: v_accvgpr_mov_b32 a78, a28
-; CHECK-NEXT: v_accvgpr_mov_b32 a77, a27
-; CHECK-NEXT: v_accvgpr_mov_b32 a76, a26
-; CHECK-NEXT: v_accvgpr_mov_b32 a75, a25
-; CHECK-NEXT: v_accvgpr_mov_b32 a74, a24
-; CHECK-NEXT: v_accvgpr_mov_b32 a73, a23
-; CHECK-NEXT: v_accvgpr_mov_b32 a72, a22
-; CHECK-NEXT: v_accvgpr_mov_b32 a71, a21
-; CHECK-NEXT: v_accvgpr_mov_b32 a70, a20
-; CHECK-NEXT: v_accvgpr_mov_b32 a69, a19
-; CHECK-NEXT: v_accvgpr_mov_b32 a68, a18
+; CHECK-NEXT: v_accvgpr_mov_b32 a79, a13
+; CHECK-NEXT: v_accvgpr_mov_b32 a78, a12
+; CHECK-NEXT: v_accvgpr_mov_b32 a77, a11
+; CHECK-NEXT: v_accvgpr_mov_b32 a76, a10
+; CHECK-NEXT: v_accvgpr_mov_b32 a75, a9
+; CHECK-NEXT: v_accvgpr_mov_b32 a74, a8
+; CHECK-NEXT: v_accvgpr_mov_b32 a73, a7
+; CHECK-NEXT: v_accvgpr_mov_b32 a72, a6
+; CHECK-NEXT: v_accvgpr_mov_b32 a71, a5
+; CHECK-NEXT: v_accvgpr_mov_b32 a70, a4
+; CHECK-NEXT: v_accvgpr_mov_b32 a69, a3
+; CHECK-NEXT: v_accvgpr_mov_b32 a68, a2
; CHECK-NEXT: v_mfma_f32_32x32x8f16 v[130:145], v[24:25], v[24:25], v[130:145]
-; CHECK-NEXT: v_accvgpr_mov_b32 a67, a17
-; CHECK-NEXT: v_accvgpr_mov_b32 a66, a16
+; CHECK-NEXT: v_accvgpr_mov_b32 a67, a1
+; CHECK-NEXT: v_accvgpr_mov_b32 a66, a0
; CHECK-NEXT: v_mov_b32_e32 v97, v3
; CHECK-NEXT: v_accvgpr_read_b32 v2, a64
; CHECK-NEXT: v_accvgpr_mov_b32 a50, a98
@@ -415,7 +415,7 @@ define amdgpu_kernel void @multi_store_spill_slot() #0 {
; CHECK-NEXT: v_mov_b32_e32 v127, v3
; CHECK-NEXT: v_mov_b32_e32 v128, v3
; CHECK-NEXT: v_mov_b32_e32 v129, v3
-; CHECK-NEXT: v_accvgpr_mov_b32 a17, a16
+; CHECK-NEXT: v_accvgpr_mov_b32 a1, a0
; CHECK-NEXT: v_accvgpr_mov_b32 a115, a114
; CHECK-NEXT: v_mfma_f32_32x32x8f16 v[98:113], v[24:25], v[24:25], v[98:113]
; CHECK-NEXT: v_mov_b32_e32 v243, v242
@@ -433,7 +433,7 @@ define amdgpu_kernel void @multi_store_spill_slot() #0 {
; CHECK-NEXT: s_nop 1
; CHECK-NEXT: v_mov_b32_e32 v145, v130
; CHECK-NEXT: v_mfma_f32_32x32x8f16 a[34:49], v[24:25], v[22:23], a[34:49]
-; CHECK-NEXT: v_accvgpr_mov_b32 a0, a82
+; CHECK-NEXT: v_accvgpr_mov_b32 a16, a82
; CHECK-NEXT: v_mfma_f32_32x32x8f16 a[50:65], v[24:25], v[24:25], a[50:65]
; CHECK-NEXT: s_nop 15
; CHECK-NEXT: s_nop 0
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-multi-store.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-multi-store.ll
index 21dd9ff425623..4ccf8e67796f7 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-multi-store.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-spill-multi-store.ll
@@ -9,42 +9,47 @@
; the verifier would error out. Check that the unspilled interval was split
; into separate components.
+; The accumulators are initialized from %init rather than from 0.0 on purpose.
+; A constant PHI operand shares one register between all the PHIs that use it,
+; in a class chosen by the constant, which changes the spilling enough that the
+; pass no longer unspills anything and the test stops testing anything.
+
; CHECK: Split unspilled interval into {{[0-9]+}} components
-define amdgpu_kernel void @multi_store_spill_slot() #0 {
+define amdgpu_kernel void @multi_store_spill_slot(float %init) #0 {
entry:
br label %do.body
do.body:
- %c_block_tile.sroa.37.0 = phi float [ 0.000000e+00, %entry ], [ %c_block_tile.sroa.70.0, %do.body ]
- %c_block_tile.sroa.70.0 = phi float [ 0.000000e+00, %entry ], [ %ext0, %do.body ]
- %c_block_tile.sroa.961.0 = phi float [ 0.000000e+00, %entry ], [ %ext1, %do.body ]
- %c_block_tile.sroa.994.0 = phi float [ 0.000000e+00, %entry ], [ %ext2, %do.body ]
- %c_block_tile.sroa.1060.0 = phi float [ 0.000000e+00, %entry ], [ %ext3, %do.body ]
- %c_block_tile.sroa.1093.0 = phi float [ 0.000000e+00, %entry ], [ %c_block_tile.sroa.1060.0, %do.body ]
- %c_block_tile.sroa.1588.0 = phi float [ 0.000000e+00, %entry ], [ %ext4, %do.body ]
- %c_block_tile.sroa.1687.0 = phi float [ 0.000000e+00, %entry ], [ %ext5, %do.body ]
- %c_block_tile.sroa.2578.0 = phi float [ 0.000000e+00, %entry ], [ %ext6, %do.body ]
- %c_block_tile.sroa.2611.0 = phi float [ 0.000000e+00, %entry ], [ %ext7, %do.body ]
- %c_block_tile.sroa.2644.0 = phi float [ 0.000000e+00, %entry ], [ %ext8, %do.body ]
- %c_block_tile.sroa.2677.0 = phi float [ 0.000000e+00, %entry ], [ %ext9, %do.body ]
- %c_block_tile.sroa.3568.0 = phi float [ 0.000000e+00, %entry ], [ %ext10, %do.body ]
- %c_block_tile.sroa.3634.0 = phi float [ 0.000000e+00, %entry ], [ %ext11, %do.body ]
- %c_block_tile.sroa.3898.0 = phi float [ 0.000000e+00, %entry ], [ %ext12, %do.body ]
- %c_block_tile.sroa.3931.0 = phi float [ 0.000000e+00, %entry ], [ %ext13, %do.body ]
- %c_block_tile.sroa.4624.0 = phi float [ 0.000000e+00, %entry ], [ %ext14, %do.body ]
- %c_block_tile.sroa.4657.0 = phi float [ 0.000000e+00, %entry ], [ %ext15, %do.body ]
- %c_block_tile.sroa.5185.0 = phi float [ 0.000000e+00, %entry ], [ %ext16, %do.body ]
- %c_block_tile.sroa.5218.0 = phi float [ 0.000000e+00, %entry ], [ %ext17, %do.body ]
- %c_block_tile.sroa.5746.0 = phi float [ 0.000000e+00, %entry ], [ %ext18, %do.body ]
- %c_block_tile.sroa.5779.0 = phi float [ 0.000000e+00, %entry ], [ %c_block_tile.sroa.5746.0, %do.body ]
- %c_block_tile.sroa.5812.0 = phi float [ 0.000000e+00, %entry ], [ %ext19, %do.body ]
- %c_block_tile.sroa.6373.0 = phi float [ 0.000000e+00, %entry ], [ %c_block_tile.sroa.6406.0, %do.body ]
- %c_block_tile.sroa.6406.0 = phi float [ 0.000000e+00, %entry ], [ %ext20, %do.body ]
- %c_block_tile.sroa.7297.0 = phi float [ 0.000000e+00, %entry ], [ %ext21, %do.body ]
- %c_block_tile.sroa.7363.0 = phi float [ 0.000000e+00, %entry ], [ %ext22, %do.body ]
- %c_block_tile.sroa.7396.0 = phi float [ 0.000000e+00, %entry ], [ %ext23, %do.body ]
- %c_block_tile.sroa.7429.0 = phi float [ 0.000000e+00, %entry ], [ %ext24, %do.body ]
+ %c_block_tile.sroa.37.0 = phi float [ %init, %entry ], [ %c_block_tile.sroa.70.0, %do.body ]
+ %c_block_tile.sroa.70.0 = phi float [ %init, %entry ], [ %ext0, %do.body ]
+ %c_block_tile.sroa.961.0 = phi float [ %init, %entry ], [ %ext1, %do.body ]
+ %c_block_tile.sroa.994.0 = phi float [ %init, %entry ], [ %ext2, %do.body ]
+ %c_block_tile.sroa.1060.0 = phi float [ %init, %entry ], [ %ext3, %do.body ]
+ %c_block_tile.sroa.1093.0 = phi float [ %init, %entry ], [ %c_block_tile.sroa.1060.0, %do.body ]
+ %c_block_tile.sroa.1588.0 = phi float [ %init, %entry ], [ %ext4, %do.body ]
+ %c_block_tile.sroa.1687.0 = phi float [ %init, %entry ], [ %ext5, %do.body ]
+ %c_block_tile.sroa.2578.0 = phi float [ %init, %entry ], [ %ext6, %do.body ]
+ %c_block_tile.sroa.2611.0 = phi float [ %init, %entry ], [ %ext7, %do.body ]
+ %c_block_tile.sroa.2644.0 = phi float [ %init, %entry ], [ %ext8, %do.body ]
+ %c_block_tile.sroa.2677.0 = phi float [ %init, %entry ], [ %ext9, %do.body ]
+ %c_block_tile.sroa.3568.0 = phi float [ %init, %entry ], [ %ext10, %do.body ]
+ %c_block_tile.sroa.3634.0 = phi float [ %init, %entry ], [ %ext11, %do.body ]
+ %c_block_tile.sroa.3898.0 = phi float [ %init, %entry ], [ %ext12, %do.body ]
+ %c_block_tile.sroa.3931.0 = phi float [ %init, %entry ], [ %ext13, %do.body ]
+ %c_block_tile.sroa.4624.0 = phi float [ %init, %entry ], [ %ext14, %do.body ]
+ %c_block_tile.sroa.4657.0 = phi float [ %init, %entry ], [ %ext15, %do.body ]
+ %c_block_tile.sroa.5185.0 = phi float [ %init, %entry ], [ %ext16, %do.body ]
+ %c_block_tile.sroa.5218.0 = phi float [ %init, %entry ], [ %ext17, %do.body ]
+ %c_block_tile.sroa.5746.0 = phi float [ %init, %entry ], [ %ext18, %do.body ]
+ %c_block_tile.sroa.5779.0 = phi float [ %init, %entry ], [ %c_block_tile.sroa.5746.0, %do.body ]
+ %c_block_tile.sroa.5812.0 = phi float [ %init, %entry ], [ %ext19, %do.body ]
+ %c_block_tile.sroa.6373.0 = phi float [ %init, %entry ], [ %c_block_tile.sroa.6406.0, %do.body ]
+ %c_block_tile.sroa.6406.0 = phi float [ %init, %entry ], [ %ext20, %do.body ]
+ %c_block_tile.sroa.7297.0 = phi float [ %init, %entry ], [ %ext21, %do.body ]
+ %c_block_tile.sroa.7363.0 = phi float [ %init, %entry ], [ %ext22, %do.body ]
+ %c_block_tile.sroa.7396.0 = phi float [ %init, %entry ], [ %ext23, %do.body ]
+ %c_block_tile.sroa.7429.0 = phi float [ %init, %entry ], [ %ext24, %do.body ]
%v0 = insertelement <16 x float> zeroinitializer, float %c_block_tile.sroa.37.0, i64 0
%v1 = insertelement <16 x float> %v0, float %c_block_tile.sroa.70.0, i64 0
%0 = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x8f16(<4 x half> zeroinitializer, <4 x half> zeroinitializer, <16 x float> %v1, i32 0, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/scheduler-rp-calc-one-successor-two-predecessors-bug.ll b/llvm/test/CodeGen/AMDGPU/scheduler-rp-calc-one-successor-two-predecessors-bug.ll
index c8501299785e0..cb46afea40460 100644
--- a/llvm/test/CodeGen/AMDGPU/scheduler-rp-calc-one-successor-two-predecessors-bug.ll
+++ b/llvm/test/CodeGen/AMDGPU/scheduler-rp-calc-one-successor-two-predecessors-bug.ll
@@ -46,7 +46,7 @@ define amdgpu_ps void @_amdgpu_ps_main(float %arg) {
; GFX900-NEXT: s_mov_b64 exec, 0
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: v_mov_b32_e32 v1, 0
-; GFX900-NEXT: v_mov_b32_e32 v2, v1
+; GFX900-NEXT: v_mov_b32_e32 v2, 0
; GFX900-NEXT: .LBB0_5: ; %bb6
; GFX900-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX900-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
index 6d8321014cd25..31766b81fcd5a 100644
--- a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
@@ -1932,6 +1932,7 @@ define amdgpu_kernel void @sdwa_crash_inlineasm_def() #0 {
; NOSDWA: ; %bb.0: ; %bb
; NOSDWA-NEXT: s_mov_b32 s0, 0xffff
; NOSDWA-NEXT: s_and_b64 vcc, exec, -1
+; NOSDWA-NEXT: ; implicit-def: $vgpr0_vgpr1
; NOSDWA-NEXT: .LBB21_1: ; %bb1
; NOSDWA-NEXT: ; =>This Inner Loop Header: Depth=1
; NOSDWA-NEXT: ;;#ASMSTART
@@ -1949,6 +1950,7 @@ define amdgpu_kernel void @sdwa_crash_inlineasm_def() #0 {
; GFX89: ; %bb.0: ; %bb
; GFX89-NEXT: s_mov_b32 s0, 0xffff
; GFX89-NEXT: s_and_b64 vcc, exec, -1
+; GFX89-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX89-NEXT: .LBB21_1: ; %bb1
; GFX89-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX89-NEXT: ;;#ASMSTART
@@ -1966,6 +1968,7 @@ define amdgpu_kernel void @sdwa_crash_inlineasm_def() #0 {
; GFX9: ; %bb.0: ; %bb
; GFX9-NEXT: s_mov_b32 s0, 0xffff
; GFX9-NEXT: s_and_b64 vcc, exec, -1
+; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9-NEXT: .LBB21_1: ; %bb1
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: ;;#ASMSTART
@@ -1983,6 +1986,7 @@ define amdgpu_kernel void @sdwa_crash_inlineasm_def() #0 {
; GFX10: ; %bb.0: ; %bb
; GFX10-NEXT: s_mov_b32 s0, 0xffff
; GFX10-NEXT: s_mov_b32 vcc_lo, exec_lo
+; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX10-NEXT: .LBB21_1: ; %bb1
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: ;;#ASMSTART
diff --git a/llvm/test/CodeGen/AMDGPU/set-inactive-wwm-overwrite.ll b/llvm/test/CodeGen/AMDGPU/set-inactive-wwm-overwrite.ll
index fa9994ba2544d..0d230d0509c43 100644
--- a/llvm/test/CodeGen/AMDGPU/set-inactive-wwm-overwrite.ll
+++ b/llvm/test/CodeGen/AMDGPU/set-inactive-wwm-overwrite.ll
@@ -4,8 +4,8 @@
define amdgpu_cs void @if_then(ptr addrspace(8) inreg %input, ptr addrspace(8) inreg %output, <3 x i32> %LocalInvocationId) {
; GCN-LABEL: if_then:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GCN-NEXT: ; %bb.1: ; %.bb0
; GCN-NEXT: v_mov_b32_e32 v3, 1
@@ -60,8 +60,8 @@ define amdgpu_cs void @if_then(ptr addrspace(8) inreg %input, ptr addrspace(8) i
define amdgpu_cs void @if_else_vgpr_opt(ptr addrspace(8) inreg %input, ptr addrspace(8) inreg %output, <3 x i32> %LocalInvocationId) {
; GCN-LABEL: if_else_vgpr_opt:
; GCN: ; %bb.0: ; %.entry
-; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GCN-NEXT: ; %bb.1: ; %.bb0
; GCN-NEXT: v_mov_b32_e32 v3, 1
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-control-flow.ll b/llvm/test/CodeGen/AMDGPU/sgpr-control-flow.ll
index a32cfd0d45950..27eddb74b3163 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-control-flow.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-control-flow.ll
@@ -184,7 +184,8 @@ define amdgpu_kernel void @sgpr_if_else_valu_cmp_phi_br(ptr addrspace(1) %out, p
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cmp_gt_i32_e64 s[8:9], 0, v0
+; SI-NEXT: v_cmp_gt_i32_e32 vcc, 0, v0
+; SI-NEXT: s_and_b64 s[8:9], vcc, exec
; SI-NEXT: ; implicit-def: $vgpr0
; SI-NEXT: .LBB3_2: ; %Flow
; SI-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-scavenge-fi-stack-id.ll b/llvm/test/CodeGen/AMDGPU/sgpr-scavenge-fi-stack-id.ll
index 883fd8188bbf7..38c33bcb922ca 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-scavenge-fi-stack-id.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-scavenge-fi-stack-id.ll
@@ -44,13 +44,13 @@ define void @sgpr_scavenge_fi_stack_id(double %input, i1 %enter_fma_path, i1 %re
; CHECK-NEXT: s_and_b64 s[44:45], exec, s[44:45]
; CHECK-NEXT: s_or_b64 s[20:21], s[44:45], s[20:21]
; CHECK-NEXT: s_andn2_b64 exec, exec, s[20:21]
-; CHECK-NEXT: s_cbranch_execz .LBB0_21
+; CHECK-NEXT: s_cbranch_execz .LBB0_22
; CHECK-NEXT: .LBB0_2: ; %outer_latch
; CHECK-NEXT: ; =>This Loop Header: Depth=1
; CHECK-NEXT: ; Child Loop BB0_4 Depth 2
; CHECK-NEXT: ; Child Loop BB0_8 Depth 3
-; CHECK-NEXT: ; Child Loop BB0_11 Depth 4
-; CHECK-NEXT: ; Child Loop BB0_13 Depth 5
+; CHECK-NEXT: ; Child Loop BB0_12 Depth 4
+; CHECK-NEXT: ; Child Loop BB0_14 Depth 5
; CHECK-NEXT: s_mov_b64 s[46:47], 0
; CHECK-NEXT: ; implicit-def: $sgpr44_sgpr45
; CHECK-NEXT: ; implicit-def: $sgpr56_sgpr57
@@ -69,12 +69,12 @@ define void @sgpr_scavenge_fi_stack_id(double %input, i1 %enter_fma_path, i1 %re
; CHECK-NEXT: ; Parent Loop BB0_2 Depth=1
; CHECK-NEXT: ; => This Loop Header: Depth=2
; CHECK-NEXT: ; Child Loop BB0_8 Depth 3
-; CHECK-NEXT: ; Child Loop BB0_11 Depth 4
-; CHECK-NEXT: ; Child Loop BB0_13 Depth 5
+; CHECK-NEXT: ; Child Loop BB0_12 Depth 4
+; CHECK-NEXT: ; Child Loop BB0_14 Depth 5
; CHECK-NEXT: s_mov_b64 s[58:59], -1
; CHECK-NEXT: s_mov_b64 s[62:63], -1
; CHECK-NEXT: s_and_saveexec_b64 s[60:61], s[10:11]
-; CHECK-NEXT: s_cbranch_execz .LBB0_19
+; CHECK-NEXT: s_cbranch_execz .LBB0_20
; CHECK-NEXT: ; %bb.5: ; %fma_setup
; CHECK-NEXT: ; in Loop: Header=BB0_4 Depth=2
; CHECK-NEXT: v_fma_f64 v[2:3], v[0:1], s[24:25], 0
@@ -102,13 +102,13 @@ define void @sgpr_scavenge_fi_stack_id(double %input, i1 %enter_fma_path, i1 %re
; CHECK-NEXT: s_and_b64 s[78:79], s[76:77], exec
; CHECK-NEXT: s_or_b64 s[72:73], s[72:73], s[78:79]
; CHECK-NEXT: s_andn2_b64 exec, exec, s[62:63]
-; CHECK-NEXT: s_cbranch_execz .LBB0_18
+; CHECK-NEXT: s_cbranch_execz .LBB0_19
; CHECK-NEXT: .LBB0_8: ; %sgpr_loop_header
; CHECK-NEXT: ; Parent Loop BB0_2 Depth=1
; CHECK-NEXT: ; Parent Loop BB0_4 Depth=2
; CHECK-NEXT: ; => This Loop Header: Depth=3
-; CHECK-NEXT: ; Child Loop BB0_11 Depth 4
-; CHECK-NEXT: ; Child Loop BB0_13 Depth 5
+; CHECK-NEXT: ; Child Loop BB0_12 Depth 4
+; CHECK-NEXT: ; Child Loop BB0_14 Depth 5
; CHECK-NEXT: s_or_b64 s[76:77], s[76:77], exec
; CHECK-NEXT: s_or_b64 s[74:75], s[74:75], exec
; CHECK-NEXT: s_and_saveexec_b64 s[78:79], s[12:13]
@@ -117,9 +117,13 @@ define void @sgpr_scavenge_fi_stack_id(double %input, i1 %enter_fma_path, i1 %re
; CHECK-NEXT: ; in Loop: Header=BB0_8 Depth=3
; CHECK-NEXT: s_mov_b64 s[90:91], 0
; CHECK-NEXT: ; implicit-def: $sgpr88_sgpr89
-; CHECK-NEXT: s_branch .LBB0_11
-; CHECK-NEXT: .LBB0_10: ; %Flow7
-; CHECK-NEXT: ; in Loop: Header=BB0_11 Depth=4
+; CHECK-NEXT: s_branch .LBB0_12
+; CHECK-NEXT: .LBB0_10: ; %sgpr_loop_merge
+; CHECK-NEXT: ; in Loop: Header=BB0_12 Depth=4
+; CHECK-NEXT: s_mov_b64 s[94:95], 0
+; CHECK-NEXT: s_orn2_b64 s[92:93], s[22:23], exec
+; CHECK-NEXT: .LBB0_11: ; %Flow7
+; CHECK-NEXT: ; in Loop: Header=BB0_12 Depth=4
; CHECK-NEXT: s_xor_b64 s[94:95], s[94:95], -1
; CHECK-NEXT: s_and_b64 s[92:93], exec, s[92:93]
; CHECK-NEXT: s_or_b64 s[90:91], s[92:93], s[90:91]
@@ -127,17 +131,17 @@ define void @sgpr_scavenge_fi_stack_id(double %input, i1 %enter_fma_path, i1 %re
; CHECK-NEXT: s_and_b64 s[92:93], s[94:95], exec
; CHECK-NEXT: s_or_b64 s[88:89], s[88:89], s[92:93]
; CHECK-NEXT: s_andn2_b64 exec, exec, s[90:91]
-; CHECK-NEXT: s_cbranch_execz .LBB0_16
-; CHECK-NEXT: .LBB0_11: ; %sgpr_pressure_loop
+; CHECK-NEXT: s_cbranch_execz .LBB0_17
+; CHECK-NEXT: .LBB0_12: ; %sgpr_pressure_loop
; CHECK-NEXT: ; Parent Loop BB0_2 Depth=1
; CHECK-NEXT: ; Parent Loop BB0_4 Depth=2
; CHECK-NEXT: ; Parent Loop BB0_8 Depth=3
; CHECK-NEXT: ; => This Loop Header: Depth=4
-; CHECK-NEXT: ; Child Loop BB0_13 Depth 5
+; CHECK-NEXT: ; Child Loop BB0_14 Depth 5
; CHECK-NEXT: s_and_saveexec_b64 s[92:93], s[8:9]
-; CHECK-NEXT: s_cbranch_execz .LBB0_14
-; CHECK-NEXT: ; %bb.12: ; %inner_stack_loop.preheader
-; CHECK-NEXT: ; in Loop: Header=BB0_11 Depth=4
+; CHECK-NEXT: s_cbranch_execz .LBB0_15
+; CHECK-NEXT: ; %bb.13: ; %inner_stack_loop.preheader
+; CHECK-NEXT: ; in Loop: Header=BB0_12 Depth=4
; CHECK-NEXT: v_cndmask_b32_e64 v4, v3, 0, s[4:5]
; CHECK-NEXT: v_cndmask_b32_e64 v7, v2, 0, s[4:5]
; CHECK-NEXT: v_cndmask_b32_e64 v5, v4, 0, s[14:15]
@@ -147,11 +151,11 @@ define void @sgpr_scavenge_fi_stack_id(double %input, i1 %enter_fma_path, i1 %re
; CHECK-NEXT: s_mov_b32 vcc_lo, 0
; CHECK-NEXT: s_mov_b64 s[94:95], 0
; CHECK-NEXT: v_mul_f64 v[4:5], v[4:5], s[42:43]
-; CHECK-NEXT: .LBB0_13: ; %inner_stack_loop
+; CHECK-NEXT: .LBB0_14: ; %inner_stack_loop
; CHECK-NEXT: ; Parent Loop BB0_2 Depth=1
; CHECK-NEXT: ; Parent Loop BB0_4 Depth=2
; CHECK-NEXT: ; Parent Loop BB0_8 Depth=3
-; CHECK-NEXT: ; Parent Loop BB0_11 Depth=4
+; CHECK-NEXT: ; Parent Loop BB0_12 Depth=4
; CHECK-NEXT: ; => This Inner Loop Header: Depth=5
; CHECK-NEXT: s_lshr_b32 s30, s32, 6
; CHECK-NEXT: s_lshl3_add_u32 vcc_hi, vcc_lo, s30
@@ -169,19 +173,16 @@ define void @sgpr_scavenge_fi_stack_id(double %input, i1 %enter_fma_path, i1 %re
; CHECK-NEXT: buffer_store_dword v6, v7, s[0:3], 0 offen
; CHECK-NEXT: buffer_store_dword v6, v7, s[0:3], 0 offen offset:4
; CHECK-NEXT: s_andn2_b64 exec, exec, s[94:95]
-; CHECK-NEXT: s_cbranch_execnz .LBB0_13
-; CHECK-NEXT: .LBB0_14: ; %Flow
-; CHECK-NEXT: ; in Loop: Header=BB0_11 Depth=4
+; CHECK-NEXT: s_cbranch_execnz .LBB0_14
+; CHECK-NEXT: .LBB0_15: ; %Flow
+; CHECK-NEXT: ; in Loop: Header=BB0_12 Depth=4
; CHECK-NEXT: s_or_b64 exec, exec, s[92:93]
; CHECK-NEXT: s_mov_b64 s[92:93], -1
+; CHECK-NEXT: s_cbranch_execnz .LBB0_10
+; CHECK-NEXT: ; %bb.16: ; in Loop: Header=BB0_12 Depth=4
; CHECK-NEXT: s_mov_b64 s[94:95], -1
-; CHECK-NEXT: s_cbranch_execz .LBB0_10
-; CHECK-NEXT: ; %bb.15: ; %sgpr_loop_merge
-; CHECK-NEXT: ; in Loop: Header=BB0_11 Depth=4
-; CHECK-NEXT: s_mov_b64 s[94:95], 0
-; CHECK-NEXT: s_orn2_b64 s[92:93], s[22:23], exec
-; CHECK-NEXT: s_branch .LBB0_10
-; CHECK-NEXT: .LBB0_16: ; %loop.exit.guard6
+; CHECK-NEXT: s_branch .LBB0_11
+; CHECK-NEXT: .LBB0_17: ; %loop.exit.guard6
; CHECK-NEXT: ; in Loop: Header=BB0_8 Depth=3
; CHECK-NEXT: s_or_b64 exec, exec, s[90:91]
; CHECK-NEXT: s_mov_b64 s[90:91], -1
@@ -189,27 +190,27 @@ define void @sgpr_scavenge_fi_stack_id(double %input, i1 %enter_fma_path, i1 %re
; CHECK-NEXT: s_and_saveexec_b64 s[94:95], s[88:89]
; CHECK-NEXT: s_xor_b64 s[88:89], exec, s[94:95]
; CHECK-NEXT: s_cbranch_execz .LBB0_6
-; CHECK-NEXT: ; %bb.17: ; %after_sgpr_loop
+; CHECK-NEXT: ; %bb.18: ; %after_sgpr_loop
; CHECK-NEXT: ; in Loop: Header=BB0_8 Depth=3
; CHECK-NEXT: s_mov_b64 s[92:93], exec
; CHECK-NEXT: s_orn2_b64 s[90:91], s[18:19], exec
; CHECK-NEXT: s_branch .LBB0_6
-; CHECK-NEXT: .LBB0_18: ; %loop.exit.guard5
+; CHECK-NEXT: .LBB0_19: ; %loop.exit.guard5
; CHECK-NEXT: ; in Loop: Header=BB0_4 Depth=2
; CHECK-NEXT: s_or_b64 exec, exec, s[62:63]
; CHECK-NEXT: s_orn2_b64 s[62:63], s[72:73], exec
-; CHECK-NEXT: .LBB0_19: ; %Flow10
+; CHECK-NEXT: .LBB0_20: ; %Flow10
; CHECK-NEXT: ; in Loop: Header=BB0_4 Depth=2
; CHECK-NEXT: s_or_b64 exec, exec, s[60:61]
; CHECK-NEXT: s_or_b64 s[56:57], s[56:57], exec
; CHECK-NEXT: s_and_saveexec_b64 s[60:61], s[62:63]
; CHECK-NEXT: s_cbranch_execz .LBB0_3
-; CHECK-NEXT: ; %bb.20: ; %outer_exit
+; CHECK-NEXT: ; %bb.21: ; %outer_exit
; CHECK-NEXT: ; in Loop: Header=BB0_4 Depth=2
; CHECK-NEXT: s_andn2_b64 s[56:57], s[56:57], exec
; CHECK-NEXT: s_orn2_b64 s[58:59], s[16:17], exec
; CHECK-NEXT: s_branch .LBB0_3
-; CHECK-NEXT: .LBB0_21: ; %DummyReturnBlock
+; CHECK-NEXT: .LBB0_22: ; %DummyReturnBlock
; CHECK-NEXT: s_or_b64 exec, exec, s[20:21]
; CHECK-NEXT: v_readlane_b32 s30, v10, 0
; CHECK-NEXT: v_readlane_b32 s31, v10, 1
diff --git a/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll b/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll
index ccaffb0696df1..24ad6af991836 100644
--- a/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/sink-fmul-fadd.ll
@@ -26,9 +26,9 @@ define float @fma_in_loop_f32(float %a, float %b, i32 %n) {
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mul_f32_e32 v1, v0, v1
-; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: .LBB0_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_add_i32 s6, s6, 1
@@ -45,9 +45,9 @@ define float @fma_in_loop_f32(float %a, float %b, i32 %n) {
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: .LBB0_1: ; %loop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_add_i32 s6, s6, 1
@@ -117,9 +117,9 @@ define float @fsub_in_loop_f32(float %a, float %b, i32 %n) {
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mul_f32_e32 v1, v0, v1
-; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: .LBB1_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_add_i32 s6, s6, 1
@@ -136,9 +136,9 @@ define float @fsub_in_loop_f32(float %a, float %b, i32 %n) {
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: .LBB1_1: ; %loop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_add_i32 s6, s6, 1
@@ -208,9 +208,9 @@ define half @fma_in_loop_f16(half %a, half %b, i32 %n) {
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: .LBB2_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_add_i32 s6, s6, 1
@@ -227,9 +227,9 @@ define half @fma_in_loop_f16(half %a, half %b, i32 %n) {
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: .LBB2_1: ; %loop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_add_i32 s6, s6, 1
@@ -300,9 +300,9 @@ define double @fma_in_loop_f64(double %a, double %b, i32 %n) {
; GFX8-LABEL: fma_in_loop_f64:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: v_mov_b32_e32 v5, 0
; GFX8-NEXT: v_mov_b32_e32 v6, 0
-; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB3_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -321,9 +321,9 @@ define double @fma_in_loop_f64(double %a, double %b, i32 %n) {
; GFX9-LABEL: fma_in_loop_f64:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: v_mov_b32_e32 v5, 0
; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: .LBB3_1: ; %loop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -399,9 +399,9 @@ define float @mad_in_loop_f32(float %a, float %b, i32 %n) #0 {
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: .LBB4_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_add_i32 s6, s6, 1
@@ -418,9 +418,9 @@ define float @mad_in_loop_f32(float %a, float %b, i32 %n) #0 {
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: .LBB4_1: ; %loop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_add_i32 s6, s6, 1
@@ -491,9 +491,9 @@ define float @no_sink_in_loop_no_contract(float %a, float %b, i32 %n) {
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mul_f32_e32 v1, v0, v1
-; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: .LBB5_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_add_i32 s6, s6, 1
@@ -510,9 +510,9 @@ define float @no_sink_in_loop_no_contract(float %a, float %b, i32 %n) {
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mul_f32_e32 v1, v0, v1
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: .LBB5_1: ; %loop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_add_i32 s6, s6, 1
@@ -584,9 +584,9 @@ define float @no_sink_in_loop_multi_use(float %a, float %b, i32 %n) {
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v1, 0
; GFX8-NEXT: .LBB6_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_add_i32 s6, s6, 1
@@ -604,9 +604,9 @@ define float @no_sink_in_loop_multi_use(float %a, float %b, i32 %n) {
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: .LBB6_1: ; %loop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_add_i32 s6, s6, 1
@@ -662,9 +662,9 @@ define half @mad_in_loop_f16(half %a, half %b, i32 %n) #0 {
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: .LBB7_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_add_i32 s6, s6, 1
@@ -681,9 +681,9 @@ define half @mad_in_loop_f16(half %a, half %b, i32 %n) #0 {
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: .LBB7_1: ; %loop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_add_i32 s6, s6, 1
@@ -736,9 +736,9 @@ define <2 x half> @no_fma_in_loop_v2f16(<2 x half> %a, <2 x half> %b, i32 %n) #0
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: .LBB8_1: ; %loop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v0
@@ -759,9 +759,9 @@ define <2 x half> @no_fma_in_loop_v2f16(<2 x half> %a, <2 x half> %b, i32 %n) #0
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_pk_mul_f16 v1, v3, v1
; GFX9-NEXT: .LBB8_1: ; %loop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -813,9 +813,9 @@ define <2 x half> @fma_in_loop_v2f16_denormals(<2 x half> %a, <2 x half> %b, i32
; GFX8-LABEL: fma_in_loop_v2f16_denormals:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v3, 0
; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; GFX8-NEXT: .LBB9_1: ; %loop
@@ -839,9 +839,9 @@ define <2 x half> @fma_in_loop_v2f16_denormals(<2 x half> %a, <2 x half> %b, i32
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: .LBB9_1: ; %loop
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_add_i32 s6, s6, 1
@@ -894,9 +894,9 @@ define bfloat @fma_in_loop_bf16(bfloat %a, bfloat %b, i32 %n) {
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: s_mov_b32 s6, 0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX8-NEXT: .LBB10_1: ; %loop
@@ -923,9 +923,9 @@ define bfloat @fma_in_loop_bf16(bfloat %a, bfloat %b, i32 %n) {
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_mov_b32 s6, 0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX9-NEXT: s_movk_i32 s7, 0x7fff
diff --git a/llvm/test/CodeGen/AMDGPU/structurize-hoist.ll b/llvm/test/CodeGen/AMDGPU/structurize-hoist.ll
index 981dd9a81243a..33f765d7f6162 100644
--- a/llvm/test/CodeGen/AMDGPU/structurize-hoist.ll
+++ b/llvm/test/CodeGen/AMDGPU/structurize-hoist.ll
@@ -87,8 +87,8 @@ define amdgpu_kernel void @test_loop_with_if( ptr %ptr, i1 %cond) #0 {
; GFX900: ; %bb.0: ; %entry
; GFX900-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX900-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX900-NEXT: v_mov_b32_e32 v5, 0
; GFX900-NEXT: s_mov_b64 s[0:1], 0
+; GFX900-NEXT: v_mov_b32_e32 v5, 0
; GFX900-NEXT: s_movk_i32 s10, 0xfe
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: s_bitcmp1_b32 s2, 0
diff --git a/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll b/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll
index 8e493437b1b19..fab9b5e45fc98 100644
--- a/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll
+++ b/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll
@@ -100,16 +100,16 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GLOBALNESS1-NEXT: s_cselect_b32 s6, 1, 0
; GLOBALNESS1-NEXT: s_cmp_lg_u32 s6, 1
-; GLOBALNESS1-NEXT: s_cbranch_scc0 .LBB1_29
+; GLOBALNESS1-NEXT: s_cbranch_scc0 .LBB1_30
; GLOBALNESS1-NEXT: .LBB1_2: ; %Flow15
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS1-NEXT: s_or_b64 exec, exec, s[4:5]
-; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], 0
+; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], 0
; GLOBALNESS1-NEXT: .LBB1_3: ; %Flow28
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[6:7]
+; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[4:5]
; GLOBALNESS1-NEXT: v_pk_mov_b32 v[58:59], v[0:1], v[0:1] op_sel:[0,1]
-; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_30
+; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_31
; GLOBALNESS1-NEXT: .LBB1_4: ; %bb5
; GLOBALNESS1-NEXT: ; =>This Loop Header: Depth=1
; GLOBALNESS1-NEXT: ; Child Loop BB1_17 Depth 2
@@ -168,10 +168,8 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: v_writelane_b32 v57, s9, 11
; GLOBALNESS1-NEXT: .LBB1_10: ; %Flow25
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], -1
; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GLOBALNESS1-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GLOBALNESS1-NEXT: s_cbranch_vccz .LBB1_3
+; GLOBALNESS1-NEXT: s_cbranch_vccz .LBB1_25
; GLOBALNESS1-NEXT: ; %bb.11: ; %baz.exit.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS1-NEXT: flat_load_dword v0, v[44:45]
@@ -180,7 +178,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: v_cmp_gt_i32_e64 s[52:53], 0, v0
; GLOBALNESS1-NEXT: v_mov_b32_e32 v0, 0
; GLOBALNESS1-NEXT: s_and_saveexec_b64 s[80:81], s[52:53]
-; GLOBALNESS1-NEXT: s_cbranch_execz .LBB1_26
+; GLOBALNESS1-NEXT: s_cbranch_execz .LBB1_27
; GLOBALNESS1-NEXT: ; %bb.12: ; %bb33.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS1-NEXT: global_load_dwordx2 v[0:1], v[44:45], off
@@ -210,7 +208,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: s_and_b64 s[4:5], s[98:99], exec
; GLOBALNESS1-NEXT: s_cselect_b32 s4, 1, 0
; GLOBALNESS1-NEXT: s_cmp_lg_u32 s4, 1
-; GLOBALNESS1-NEXT: s_cbranch_scc0 .LBB1_25
+; GLOBALNESS1-NEXT: s_cbranch_scc0 .LBB1_26
; GLOBALNESS1-NEXT: .LBB1_17: ; %bb44.i
; GLOBALNESS1-NEXT: ; Parent Loop BB1_4 Depth=1
; GLOBALNESS1-NEXT: ; => This Inner Loop Header: Depth=2
@@ -284,18 +282,22 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_17 Depth=2
; GLOBALNESS1-NEXT: global_store_dwordx2 v[44:45], v[42:43], off
; GLOBALNESS1-NEXT: s_branch .LBB1_15
-; GLOBALNESS1-NEXT: .LBB1_25: ; %Flow23
+; GLOBALNESS1-NEXT: .LBB1_25: ; in Loop: Header=BB1_4 Depth=1
+; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], -1
+; GLOBALNESS1-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GLOBALNESS1-NEXT: s_branch .LBB1_3
+; GLOBALNESS1-NEXT: .LBB1_26: ; %Flow23
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS1-NEXT: v_readlane_b32 s64, v57, 8
; GLOBALNESS1-NEXT: v_pk_mov_b32 v[0:1], 0, 0
; GLOBALNESS1-NEXT: s_mov_b32 s55, s71
; GLOBALNESS1-NEXT: v_readlane_b32 s65, v57, 9
-; GLOBALNESS1-NEXT: .LBB1_26: ; %Flow24
+; GLOBALNESS1-NEXT: .LBB1_27: ; %Flow24
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS1-NEXT: s_or_b64 exec, exec, s[80:81]
; GLOBALNESS1-NEXT: s_and_saveexec_b64 s[4:5], s[52:53]
; GLOBALNESS1-NEXT: s_cbranch_execz .LBB1_2
-; GLOBALNESS1-NEXT: ; %bb.27: ; %bb67.i
+; GLOBALNESS1-NEXT: ; %bb.28: ; %bb67.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS1-NEXT: v_readlane_b32 s6, v57, 4
; GLOBALNESS1-NEXT: v_readlane_b32 s7, v57, 5
@@ -303,23 +305,23 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: s_cselect_b32 s6, 1, 0
; GLOBALNESS1-NEXT: s_cmp_lg_u32 s6, 1
; GLOBALNESS1-NEXT: s_cbranch_scc1 .LBB1_1
-; GLOBALNESS1-NEXT: ; %bb.28: ; %bb69.i
+; GLOBALNESS1-NEXT: ; %bb.29: ; %bb69.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS1-NEXT: global_store_dwordx2 v[44:45], v[42:43], off
; GLOBALNESS1-NEXT: s_branch .LBB1_1
-; GLOBALNESS1-NEXT: .LBB1_29: ; %bb73.i
+; GLOBALNESS1-NEXT: .LBB1_30: ; %bb73.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS1-NEXT: global_store_dwordx2 v[44:45], v[42:43], off
; GLOBALNESS1-NEXT: s_branch .LBB1_2
-; GLOBALNESS1-NEXT: .LBB1_30: ; %loop.exit.guard
+; GLOBALNESS1-NEXT: .LBB1_31: ; %loop.exit.guard
; GLOBALNESS1-NEXT: v_readlane_b32 s4, v57, 10
; GLOBALNESS1-NEXT: v_readlane_b32 s5, v57, 11
; GLOBALNESS1-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GLOBALNESS1-NEXT: s_cselect_b32 s4, 1, 0
; GLOBALNESS1-NEXT: s_cmp_lg_u32 s4, 1
; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], -1
-; GLOBALNESS1-NEXT: s_cbranch_scc0 .LBB1_32
-; GLOBALNESS1-NEXT: ; %bb.31: ; %bb7.i.i
+; GLOBALNESS1-NEXT: s_cbranch_scc0 .LBB1_33
+; GLOBALNESS1-NEXT: ; %bb.32: ; %bb7.i.i
; GLOBALNESS1-NEXT: s_add_u32 s8, s48, 40
; GLOBALNESS1-NEXT: s_addc_u32 s9, s49, 0
; GLOBALNESS1-NEXT: s_getpc_b64 s[16:17]
@@ -335,12 +337,12 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: ; implicit-def: $sgpr15
; GLOBALNESS1-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], 0
-; GLOBALNESS1-NEXT: .LBB1_32: ; %Flow
+; GLOBALNESS1-NEXT: .LBB1_33: ; %Flow
; GLOBALNESS1-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GLOBALNESS1-NEXT: s_cselect_b32 s4, 1, 0
; GLOBALNESS1-NEXT: s_cmp_lg_u32 s4, 1
-; GLOBALNESS1-NEXT: s_cbranch_scc1 .LBB1_34
-; GLOBALNESS1-NEXT: ; %bb.33: ; %bb11.i.i
+; GLOBALNESS1-NEXT: s_cbranch_scc1 .LBB1_35
+; GLOBALNESS1-NEXT: ; %bb.34: ; %bb11.i.i
; GLOBALNESS1-NEXT: s_add_u32 s8, s48, 40
; GLOBALNESS1-NEXT: s_addc_u32 s9, s49, 0
; GLOBALNESS1-NEXT: s_getpc_b64 s[16:17]
@@ -355,7 +357,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: v_mov_b32_e32 v31, v41
; GLOBALNESS1-NEXT: ; implicit-def: $sgpr15
; GLOBALNESS1-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GLOBALNESS1-NEXT: .LBB1_34: ; %UnifiedUnreachableBlock
+; GLOBALNESS1-NEXT: .LBB1_35: ; %UnifiedUnreachableBlock
;
; GLOBALNESS0-LABEL: kernel:
; GLOBALNESS0: ; %bb.0: ; %bb
@@ -428,16 +430,16 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GLOBALNESS0-NEXT: s_cselect_b32 s6, 1, 0
; GLOBALNESS0-NEXT: s_cmp_lg_u32 s6, 1
-; GLOBALNESS0-NEXT: s_cbranch_scc0 .LBB1_29
+; GLOBALNESS0-NEXT: s_cbranch_scc0 .LBB1_30
; GLOBALNESS0-NEXT: .LBB1_2: ; %Flow15
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS0-NEXT: s_or_b64 exec, exec, s[4:5]
-; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], 0
+; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], 0
; GLOBALNESS0-NEXT: .LBB1_3: ; %Flow28
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[6:7]
+; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[4:5]
; GLOBALNESS0-NEXT: v_pk_mov_b32 v[58:59], v[0:1], v[0:1] op_sel:[0,1]
-; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_30
+; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_31
; GLOBALNESS0-NEXT: .LBB1_4: ; %bb5
; GLOBALNESS0-NEXT: ; =>This Loop Header: Depth=1
; GLOBALNESS0-NEXT: ; Child Loop BB1_17 Depth 2
@@ -496,10 +498,8 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: v_writelane_b32 v57, s9, 11
; GLOBALNESS0-NEXT: .LBB1_10: ; %Flow25
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], -1
; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[4:5]
-; GLOBALNESS0-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GLOBALNESS0-NEXT: s_cbranch_vccz .LBB1_3
+; GLOBALNESS0-NEXT: s_cbranch_vccz .LBB1_25
; GLOBALNESS0-NEXT: ; %bb.11: ; %baz.exit.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS0-NEXT: flat_load_dword v0, v[44:45]
@@ -508,7 +508,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: v_cmp_gt_i32_e64 s[52:53], 0, v0
; GLOBALNESS0-NEXT: v_mov_b32_e32 v0, 0
; GLOBALNESS0-NEXT: s_and_saveexec_b64 s[80:81], s[52:53]
-; GLOBALNESS0-NEXT: s_cbranch_execz .LBB1_26
+; GLOBALNESS0-NEXT: s_cbranch_execz .LBB1_27
; GLOBALNESS0-NEXT: ; %bb.12: ; %bb33.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS0-NEXT: global_load_dwordx2 v[0:1], v[44:45], off
@@ -537,7 +537,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: s_and_b64 s[4:5], s[98:99], exec
; GLOBALNESS0-NEXT: s_cselect_b32 s4, 1, 0
; GLOBALNESS0-NEXT: s_cmp_lg_u32 s4, 1
-; GLOBALNESS0-NEXT: s_cbranch_scc0 .LBB1_25
+; GLOBALNESS0-NEXT: s_cbranch_scc0 .LBB1_26
; GLOBALNESS0-NEXT: .LBB1_17: ; %bb44.i
; GLOBALNESS0-NEXT: ; Parent Loop BB1_4 Depth=1
; GLOBALNESS0-NEXT: ; => This Inner Loop Header: Depth=2
@@ -611,18 +611,22 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_17 Depth=2
; GLOBALNESS0-NEXT: global_store_dwordx2 v[44:45], v[42:43], off
; GLOBALNESS0-NEXT: s_branch .LBB1_15
-; GLOBALNESS0-NEXT: .LBB1_25: ; %Flow23
+; GLOBALNESS0-NEXT: .LBB1_25: ; in Loop: Header=BB1_4 Depth=1
+; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], -1
+; GLOBALNESS0-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GLOBALNESS0-NEXT: s_branch .LBB1_3
+; GLOBALNESS0-NEXT: .LBB1_26: ; %Flow23
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS0-NEXT: v_readlane_b32 s64, v57, 8
; GLOBALNESS0-NEXT: v_pk_mov_b32 v[0:1], 0, 0
; GLOBALNESS0-NEXT: v_readlane_b32 s65, v57, 9
-; GLOBALNESS0-NEXT: .LBB1_26: ; %Flow24
+; GLOBALNESS0-NEXT: .LBB1_27: ; %Flow24
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS0-NEXT: s_or_b64 exec, exec, s[80:81]
; GLOBALNESS0-NEXT: s_and_saveexec_b64 s[4:5], s[52:53]
; GLOBALNESS0-NEXT: s_load_dwordx4 s[52:55], s[48:49], 0x0
; GLOBALNESS0-NEXT: s_cbranch_execz .LBB1_2
-; GLOBALNESS0-NEXT: ; %bb.27: ; %bb67.i
+; GLOBALNESS0-NEXT: ; %bb.28: ; %bb67.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS0-NEXT: v_readlane_b32 s6, v57, 4
; GLOBALNESS0-NEXT: v_readlane_b32 s7, v57, 5
@@ -630,23 +634,23 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: s_cselect_b32 s6, 1, 0
; GLOBALNESS0-NEXT: s_cmp_lg_u32 s6, 1
; GLOBALNESS0-NEXT: s_cbranch_scc1 .LBB1_1
-; GLOBALNESS0-NEXT: ; %bb.28: ; %bb69.i
+; GLOBALNESS0-NEXT: ; %bb.29: ; %bb69.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS0-NEXT: global_store_dwordx2 v[44:45], v[42:43], off
; GLOBALNESS0-NEXT: s_branch .LBB1_1
-; GLOBALNESS0-NEXT: .LBB1_29: ; %bb73.i
+; GLOBALNESS0-NEXT: .LBB1_30: ; %bb73.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS0-NEXT: global_store_dwordx2 v[44:45], v[42:43], off
; GLOBALNESS0-NEXT: s_branch .LBB1_2
-; GLOBALNESS0-NEXT: .LBB1_30: ; %loop.exit.guard
+; GLOBALNESS0-NEXT: .LBB1_31: ; %loop.exit.guard
; GLOBALNESS0-NEXT: v_readlane_b32 s4, v57, 10
; GLOBALNESS0-NEXT: v_readlane_b32 s5, v57, 11
; GLOBALNESS0-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GLOBALNESS0-NEXT: s_cselect_b32 s4, 1, 0
; GLOBALNESS0-NEXT: s_cmp_lg_u32 s4, 1
; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], -1
-; GLOBALNESS0-NEXT: s_cbranch_scc0 .LBB1_32
-; GLOBALNESS0-NEXT: ; %bb.31: ; %bb7.i.i
+; GLOBALNESS0-NEXT: s_cbranch_scc0 .LBB1_33
+; GLOBALNESS0-NEXT: ; %bb.32: ; %bb7.i.i
; GLOBALNESS0-NEXT: s_add_u32 s8, s48, 40
; GLOBALNESS0-NEXT: s_addc_u32 s9, s49, 0
; GLOBALNESS0-NEXT: s_getpc_b64 s[16:17]
@@ -662,12 +666,12 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: ; implicit-def: $sgpr15
; GLOBALNESS0-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], 0
-; GLOBALNESS0-NEXT: .LBB1_32: ; %Flow
+; GLOBALNESS0-NEXT: .LBB1_33: ; %Flow
; GLOBALNESS0-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GLOBALNESS0-NEXT: s_cselect_b32 s4, 1, 0
; GLOBALNESS0-NEXT: s_cmp_lg_u32 s4, 1
-; GLOBALNESS0-NEXT: s_cbranch_scc1 .LBB1_34
-; GLOBALNESS0-NEXT: ; %bb.33: ; %bb11.i.i
+; GLOBALNESS0-NEXT: s_cbranch_scc1 .LBB1_35
+; GLOBALNESS0-NEXT: ; %bb.34: ; %bb11.i.i
; GLOBALNESS0-NEXT: s_add_u32 s8, s48, 40
; GLOBALNESS0-NEXT: s_addc_u32 s9, s49, 0
; GLOBALNESS0-NEXT: s_getpc_b64 s[16:17]
@@ -682,7 +686,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: v_mov_b32_e32 v31, v41
; GLOBALNESS0-NEXT: ; implicit-def: $sgpr15
; GLOBALNESS0-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GLOBALNESS0-NEXT: .LBB1_34: ; %UnifiedUnreachableBlock
+; GLOBALNESS0-NEXT: .LBB1_35: ; %UnifiedUnreachableBlock
bb:
store i32 0, ptr addrspace(1) null, align 4
%tmp4 = load i32, ptr addrspace(1) %arg1.global, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/undef-handling-crash-in-ra.ll b/llvm/test/CodeGen/AMDGPU/undef-handling-crash-in-ra.ll
index 1b835374f6f2d..1adf217615dfc 100644
--- a/llvm/test/CodeGen/AMDGPU/undef-handling-crash-in-ra.ll
+++ b/llvm/test/CodeGen/AMDGPU/undef-handling-crash-in-ra.ll
@@ -8,8 +8,8 @@ define amdgpu_kernel void @foo(ptr addrspace(5) %ptr5, ptr %p0, double %v0, <4 x
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_add_u32 flat_scratch_lo, s12, s17
; CHECK-NEXT: s_addc_u32 flat_scratch_hi, s13, 0
-; CHECK-NEXT: v_pk_mov_b32 v[46:47], 0, 0
-; CHECK-NEXT: flat_load_dword v42, v[46:47]
+; CHECK-NEXT: v_pk_mov_b32 v[56:57], 0, 0
+; CHECK-NEXT: flat_load_dword v42, v[56:57]
; CHECK-NEXT: s_mov_b64 s[34:35], s[8:9]
; CHECK-NEXT: s_load_dwordx4 s[64:67], s[34:35], 0x8
; CHECK-NEXT: s_load_dword s68, s[34:35], 0x0
@@ -24,7 +24,7 @@ define amdgpu_kernel void @foo(ptr addrspace(5) %ptr5, ptr %p0, double %v0, <4 x
; CHECK-NEXT: s_cselect_b32 s5, s9, 0
; CHECK-NEXT: s_cselect_b32 s6, s68, 0
; CHECK-NEXT: s_add_u32 s50, s34, 48
-; CHECK-NEXT: v_mov_b32_e32 v45, s5
+; CHECK-NEXT: v_mov_b32_e32 v47, s5
; CHECK-NEXT: s_mov_b32 s5, s4
; CHECK-NEXT: s_addc_u32 s51, s35, 0
; CHECK-NEXT: v_pk_mov_b32 v[62:63], s[4:5], s[4:5] op_sel:[0,1]
@@ -33,8 +33,8 @@ define amdgpu_kernel void @foo(ptr addrspace(5) %ptr5, ptr %p0, double %v0, <4 x
; CHECK-NEXT: s_addc_u32 s5, s5, G at gotpcrel32@hi+12
; CHECK-NEXT: s_load_dwordx2 s[54:55], s[4:5], 0x0
; CHECK-NEXT: s_mov_b32 s53, s14
-; CHECK-NEXT: v_mov_b32_e32 v44, s6
-; CHECK-NEXT: v_pk_mov_b32 v[56:57], s[64:65], s[64:65] op_sel:[0,1]
+; CHECK-NEXT: v_mov_b32_e32 v46, s6
+; CHECK-NEXT: v_pk_mov_b32 v[58:59], s[64:65], s[64:65] op_sel:[0,1]
; CHECK-NEXT: s_mov_b64 s[4:5], s[48:49]
; CHECK-NEXT: s_mov_b64 s[6:7], s[38:39]
; CHECK-NEXT: s_mov_b64 s[8:9], s[50:51]
@@ -49,12 +49,12 @@ define amdgpu_kernel void @foo(ptr addrspace(5) %ptr5, ptr %p0, double %v0, <4 x
; CHECK-NEXT: v_mov_b32_e32 v40, v0
; CHECK-NEXT: v_mov_b32_e32 v60, s66
; CHECK-NEXT: v_mov_b32_e32 v61, s67
-; CHECK-NEXT: flat_store_dwordx2 v[56:57], v[62:63]
+; CHECK-NEXT: flat_store_dwordx2 v[58:59], v[62:63]
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_swappc_b64 s[30:31], s[54:55]
-; CHECK-NEXT: flat_load_dwordx2 a[32:33], v[56:57]
-; CHECK-NEXT: v_mov_b32_e32 v58, 0
-; CHECK-NEXT: v_mov_b32_e32 v59, 0x3ff00000
+; CHECK-NEXT: flat_load_dwordx2 a[32:33], v[58:59]
+; CHECK-NEXT: v_mov_b32_e32 v44, 0
+; CHECK-NEXT: v_mov_b32_e32 v45, 0x3ff00000
; CHECK-NEXT: s_mov_b64 s[4:5], s[48:49]
; CHECK-NEXT: s_mov_b64 s[6:7], s[38:39]
; CHECK-NEXT: s_mov_b64 s[8:9], s[50:51]
@@ -63,52 +63,55 @@ define amdgpu_kernel void @foo(ptr addrspace(5) %ptr5, ptr %p0, double %v0, <4 x
; CHECK-NEXT: s_mov_b32 s13, s52
; CHECK-NEXT: s_mov_b32 s14, s33
; CHECK-NEXT: v_mov_b32_e32 v31, v40
-; CHECK-NEXT: flat_store_dwordx2 v[46:47], v[58:59]
-; CHECK-NEXT: flat_store_dwordx2 v[56:57], v[62:63]
+; CHECK-NEXT: flat_store_dwordx2 v[56:57], v[44:45]
+; CHECK-NEXT: flat_store_dwordx2 v[58:59], v[62:63]
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: ; implicit-def: $sgpr15
; CHECK-NEXT: s_swappc_b64 s[30:31], s[54:55]
-; CHECK-NEXT: flat_load_dwordx2 v[0:1], v[44:45] glc
+; CHECK-NEXT: flat_load_dwordx2 v[0:1], v[46:47] glc
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v1, s67
; CHECK-NEXT: v_mov_b32_e32 v0, s68
; CHECK-NEXT: v_cmp_lt_i32_e32 vcc, 0, v42
-; CHECK-NEXT: flat_store_dwordx2 v[56:57], a[32:33]
+; CHECK-NEXT: flat_store_dwordx2 v[58:59], a[32:33]
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_store_dwordx2 v[56:57], v[60:61]
+; CHECK-NEXT: flat_store_dwordx2 v[58:59], v[60:61]
; CHECK-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:4
-; CHECK-NEXT: buffer_store_dword v58, v0, s[0:3], 0 offen
-; CHECK-NEXT: ; implicit-def: $vgpr4
+; CHECK-NEXT: buffer_store_dword v44, v0, s[0:3], 0 offen
; CHECK-NEXT: s_and_saveexec_b64 s[4:5], vcc
; CHECK-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; CHECK-NEXT: s_cbranch_execz .LBB0_4
; CHECK-NEXT: ; %bb.1: ; %LeafBlock5
; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 1, v42
-; CHECK-NEXT: v_mov_b32_e32 v4, 0
; CHECK-NEXT: s_and_saveexec_b64 s[6:7], vcc
; CHECK-NEXT: ; %bb.2: ; %sw.bb17.i.i.i.i
-; CHECK-NEXT: v_mov_b32_e32 v4, 1
+; CHECK-NEXT: v_mov_b32_e32 v44, 1
; CHECK-NEXT: ; %bb.3: ; %Flow
; CHECK-NEXT: s_or_b64 exec, exec, s[6:7]
; CHECK-NEXT: .LBB0_4: ; %Flow8
-; CHECK-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
+; CHECK-NEXT: s_or_saveexec_b64 s[8:9], s[4:5]
; CHECK-NEXT: v_pk_mov_b32 v[0:1], v[42:43], v[42:43] op_sel:[0,1]
; CHECK-NEXT: v_pk_mov_b32 v[2:3], v[44:45], v[44:45] op_sel:[0,1]
-; CHECK-NEXT: s_xor_b64 exec, exec, s[4:5]
+; CHECK-NEXT: s_xor_b64 exec, exec, s[8:9]
; CHECK-NEXT: s_cbranch_execz .LBB0_8
; CHECK-NEXT: ; %bb.5: ; %LeafBlock
+; CHECK-NEXT: s_mov_b32 s4, 0
; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v42
; CHECK-NEXT: v_pk_mov_b32 v[0:1], v[42:43], v[42:43] op_sel:[0,1]
; CHECK-NEXT: v_pk_mov_b32 v[2:3], v[44:45], v[44:45] op_sel:[0,1]
-; CHECK-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; CHECK-NEXT: s_and_saveexec_b64 s[10:11], vcc
; CHECK-NEXT: ; %bb.6: ; %sw.bb.i.i.i.i
-; CHECK-NEXT: v_mov_b32_e32 v0, 0
+; CHECK-NEXT: s_mov_b32 s5, s4
+; CHECK-NEXT: s_mov_b32 s6, s4
+; CHECK-NEXT: s_mov_b32 s7, s4
+; CHECK-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
+; CHECK-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
; CHECK-NEXT: ; %bb.7: ; %Flow7
-; CHECK-NEXT: s_or_b64 exec, exec, s[6:7]
-; CHECK-NEXT: v_mov_b32_e32 v4, 0
+; CHECK-NEXT: s_or_b64 exec, exec, s[10:11]
+; CHECK-NEXT: v_mov_b32_e32 v44, 0
; CHECK-NEXT: .LBB0_8: ; %bb.1
-; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
-; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; CHECK-NEXT: s_or_b64 exec, exec, s[8:9]
+; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v44
; CHECK-NEXT: s_and_saveexec_b64 s[4:5], vcc
; CHECK-NEXT: s_cbranch_execz .LBB0_10
; CHECK-NEXT: ; %bb.9: ; %sw.bb.i.i.i.i.i
diff --git a/llvm/test/CodeGen/AMDGPU/uniform_branch_with_floating_point_cond.ll b/llvm/test/CodeGen/AMDGPU/uniform_branch_with_floating_point_cond.ll
index 03fd03b0a7c16..45f0a5637de6a 100644
--- a/llvm/test/CodeGen/AMDGPU/uniform_branch_with_floating_point_cond.ll
+++ b/llvm/test/CodeGen/AMDGPU/uniform_branch_with_floating_point_cond.ll
@@ -28,14 +28,14 @@ define void @test() {
; CHECK-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD killed [[COPY]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s32) from %ir.ptr)
; CHECK-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 1065353216
; CHECK-NEXT: [[V_CMP_NLT_F32_e64_:%[0-9]+]]:sreg_32 = nofpexcept V_CMP_NLT_F32_e64 0, killed [[FLAT_LOAD_DWORD]], 0, killed [[S_MOV_B32_1]], 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1092616192, implicit $exec
+ ; CHECK-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sgpr_32 = S_MOV_B32 1092616192
; CHECK-NEXT: [[SI_IF:%[0-9]+]]:sreg_32 = SI_IF killed [[V_CMP_NLT_F32_e64_]], %bb.4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.bb2:
; CHECK-NEXT: successors: %bb.4(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
; CHECK-NEXT: S_BRANCH %bb.4
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3.Flow1:
@@ -46,10 +46,10 @@ define void @test() {
; CHECK-NEXT: bb.4.bb3:
; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.6(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[V_MOV_B32_e32_]], %bb.1, [[V_MOV_B32_e32_1]], %bb.2
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[S_MOV_B32_2]], %bb.1, [[S_MOV_B32_3]], %bb.2
; CHECK-NEXT: SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
- ; CHECK-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
- ; CHECK-NEXT: [[V_CMP_EQ_F32_e64_:%[0-9]+]]:sreg_32 = nofpexcept V_CMP_EQ_F32_e64 0, [[PHI]], 0, killed [[S_MOV_B32_2]], 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sgpr_32 = S_MOV_B32 0
+ ; CHECK-NEXT: [[V_CMP_EQ_F32_e64_:%[0-9]+]]:sreg_32 = nofpexcept V_CMP_EQ_F32_e64 0, [[PHI]], 0, killed [[S_MOV_B32_4]], 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[SI_IF1:%[0-9]+]]:sreg_32 = SI_IF killed [[V_CMP_EQ_F32_e64_]], %bb.6, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; CHECK-NEXT: S_BRANCH %bb.5
; CHECK-NEXT: {{ $}}
@@ -58,9 +58,9 @@ define void @test() {
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[SI_PC_ADD_REL_OFFSET2:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @external_constant2, target-flags(amdgpu-gotprel32-hi) @external_constant2, implicit-def dead $scc
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM3:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET2]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4)
- ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1082130432, implicit $exec
- ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR killed [[V_MOV_B32_e32_2]], killed [[V_MOV_B32_e32_3]], killed [[S_LOAD_DWORDX2_IMM3]], 0, 0, implicit $exec :: (store (s32) into @external_constant2, addrspace 1)
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1082130432, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR killed [[V_MOV_B32_e32_]], killed [[V_MOV_B32_e32_1]], killed [[S_LOAD_DWORDX2_IMM3]], 0, 0, implicit $exec :: (store (s32) into @external_constant2, addrspace 1)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.6.Flow:
; CHECK-NEXT: successors: %bb.3(0x80000000)
diff --git a/llvm/test/CodeGen/AMDGPU/use-after-free-after-cleanup-failed-vreg.ll b/llvm/test/CodeGen/AMDGPU/use-after-free-after-cleanup-failed-vreg.ll
index f09bfe95b9b0e..628557f10cde4 100644
--- a/llvm/test/CodeGen/AMDGPU/use-after-free-after-cleanup-failed-vreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/use-after-free-after-cleanup-failed-vreg.ll
@@ -1,4 +1,4 @@
-; RUN: not llc -mtriple=amdgpu11.00-amd-amdhsa -stress-regalloc=4 -amdgpu-enable-rewrite-partial-reg-uses=0 -filetype=null -verify-machineinstrs %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple=amdgpu11.00-amd-amdhsa -stress-regalloc=1 -amdgpu-enable-rewrite-partial-reg-uses=0 -filetype=null -verify-machineinstrs %s 2>&1 | FileCheck %s
; CHECK: error: <unknown>:0:0: ran out of registers during register allocation in function 'f'
; CHECK-NOT: Bad machine code
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
index 0c6fcde1de385..1dbce8af58fe5 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-liverange-ir.ll
@@ -17,8 +17,8 @@ define amdgpu_ps float @else1(i32 %z, float %v) #0 {
; SI-NEXT: bb.1.Flow:
; SI-NEXT: successors: %bb.2(0x40000000), %bb.4(0x40000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %9:vgpr_32, %bb.0, %4, %bb.3
- ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, undef %14:vgpr_32, %bb.3
+ ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %13:vgpr_32, %bb.0, %4, %bb.3
+ ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, undef %15:vgpr_32, %bb.3
; SI-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32 = SI_ELSE [[SI_IF]], %bb.4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: S_BRANCH %bb.2
; SI-NEXT: {{ $}}
@@ -73,7 +73,7 @@ define amdgpu_ps float @else2(i32 %z, float %v) #0 {
; SI-NEXT: bb.1.Flow:
; SI-NEXT: successors: %bb.2(0x40000000), %bb.4(0x40000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %10:vgpr_32, %bb.0, %4, %bb.3
+ ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %15:vgpr_32, %bb.0, %4, %bb.3
; SI-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32 = SI_ELSE [[SI_IF]], %bb.4, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: S_BRANCH %bb.2
; SI-NEXT: {{ $}}
@@ -140,9 +140,9 @@ define amdgpu_ps float @else3(i32 %z, float %v, i32 inreg %bound, i32 %x0) #0 {
; SI-NEXT: bb.2.Flow:
; SI-NEXT: successors: %bb.3(0x40000000), %bb.5(0x40000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI undef %25:vgpr_32, %bb.1, %10, %bb.4
- ; SI-NEXT: [[PHI3:%[0-9]+]]:vgpr_32 = PHI undef %26:vgpr_32, %bb.1, %9, %bb.4
- ; SI-NEXT: [[PHI4:%[0-9]+]]:vgpr_32 = PHI [[PHI1]], %bb.1, undef %32:vgpr_32, %bb.4
+ ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI undef %31:vgpr_32, %bb.1, %10, %bb.4
+ ; SI-NEXT: [[PHI3:%[0-9]+]]:vgpr_32 = PHI undef %32:vgpr_32, %bb.1, %9, %bb.4
+ ; SI-NEXT: [[PHI4:%[0-9]+]]:vgpr_32 = PHI [[PHI1]], %bb.1, undef %34:vgpr_32, %bb.4
; SI-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32 = SI_ELSE [[SI_IF]], %bb.5, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: S_BRANCH %bb.3
; SI-NEXT: {{ $}}
@@ -232,10 +232,10 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: bb.1.Flow:
; SI-NEXT: successors: %bb.2(0x40000000), %bb.10(0x40000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %16:vgpr_32, %bb.0, %4, %bb.9
- ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY4]], %bb.0, undef %50:vgpr_32, %bb.9
- ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %52:vgpr_32, %bb.9
- ; SI-NEXT: [[PHI3:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %54:vgpr_32, %bb.9
+ ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %49:vgpr_32, %bb.0, %4, %bb.9
+ ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY4]], %bb.0, undef %51:vgpr_32, %bb.9
+ ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %53:vgpr_32, %bb.9
+ ; SI-NEXT: [[PHI3:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %55:vgpr_32, %bb.9
; SI-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32 = SI_ELSE [[SI_IF]], %bb.10, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: S_BRANCH %bb.2
; SI-NEXT: {{ $}}
@@ -250,8 +250,8 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: successors: %bb.4(0x80000000)
; SI-NEXT: {{ $}}
; SI-NEXT: [[PHI4:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_]], %bb.2, %41, %bb.4
- ; SI-NEXT: [[PHI5:%[0-9]+]]:vreg_64 = PHI undef %56:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2
- ; SI-NEXT: [[PHI6:%[0-9]+]]:vgpr_32 = PHI undef %58:vgpr_32, %bb.4, [[PHI1]], %bb.2
+ ; SI-NEXT: [[PHI5:%[0-9]+]]:vreg_64 = PHI undef %57:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2
+ ; SI-NEXT: [[PHI6:%[0-9]+]]:vgpr_32 = PHI undef %59:vgpr_32, %bb.4, [[PHI1]], %bb.2
; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI5]].sub0, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI5]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
@@ -288,8 +288,8 @@ define amdgpu_ps float @loop(i32 %z, float %v, i32 inreg %bound, ptr %extern_fun
; SI-NEXT: successors: %bb.8(0x80000000)
; SI-NEXT: {{ $}}
; SI-NEXT: [[PHI7:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.6, %33, %bb.8
- ; SI-NEXT: [[PHI8:%[0-9]+]]:vreg_64 = PHI undef %60:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6
- ; SI-NEXT: [[PHI9:%[0-9]+]]:vgpr_32 = PHI undef %62:vgpr_32, %bb.8, [[COPY4]], %bb.6
+ ; SI-NEXT: [[PHI8:%[0-9]+]]:vreg_64 = PHI undef %61:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6
+ ; SI-NEXT: [[PHI9:%[0-9]+]]:vgpr_32 = PHI undef %63:vgpr_32, %bb.8, [[COPY4]], %bb.6
; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI8]].sub0, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI8]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
@@ -357,9 +357,9 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: bb.1.Flow:
; SI-NEXT: successors: %bb.2(0x40000000), %bb.10(0x40000000)
; SI-NEXT: {{ $}}
- ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %16:vgpr_32, %bb.0, %4, %bb.9
- ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %51:vgpr_32, %bb.9
- ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %53:vgpr_32, %bb.9
+ ; SI-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI undef %50:vgpr_32, %bb.0, %4, %bb.9
+ ; SI-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, undef %52:vgpr_32, %bb.9
+ ; SI-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[COPY2]], %bb.0, undef %54:vgpr_32, %bb.9
; SI-NEXT: [[SI_ELSE:%[0-9]+]]:sreg_32 = SI_ELSE [[SI_IF]], %bb.10, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
; SI-NEXT: S_BRANCH %bb.2
; SI-NEXT: {{ $}}
@@ -374,7 +374,7 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: successors: %bb.4(0x80000000)
; SI-NEXT: {{ $}}
; SI-NEXT: [[PHI3:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_]], %bb.2, %42, %bb.4
- ; SI-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI undef %55:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2
+ ; SI-NEXT: [[PHI4:%[0-9]+]]:vreg_64 = PHI undef %56:vreg_64, %bb.4, [[REG_SEQUENCE]], %bb.2
; SI-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI4]].sub0, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI4]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
@@ -411,7 +411,7 @@ define amdgpu_ps float @loop_with_use(i32 %z, float %v, i32 inreg %bound, ptr %e
; SI-NEXT: successors: %bb.8(0x80000000)
; SI-NEXT: {{ $}}
; SI-NEXT: [[PHI5:%[0-9]+]]:sreg_32_xm0_xexec = PHI [[S_MOV_B32_1]], %bb.6, %34, %bb.8
- ; SI-NEXT: [[PHI6:%[0-9]+]]:vreg_64 = PHI undef %57:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6
+ ; SI-NEXT: [[PHI6:%[0-9]+]]:vreg_64 = PHI undef %58:vreg_64, %bb.8, [[REG_SEQUENCE2]], %bb.6
; SI-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI6]].sub0, implicit $exec
; SI-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[PHI6]].sub1, implicit $exec
; SI-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
diff --git a/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll b/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll
index 61d2b8458bf83..643dffbf7c168 100644
--- a/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll
+++ b/llvm/test/CodeGen/AMDGPU/vni8-across-blocks.ll
@@ -458,12 +458,13 @@ define amdgpu_kernel void @v8i8_phi_zeroinit(ptr addrspace(1) %src1, ptr addrspa
; GFX942-NEXT: s_cbranch_execz .LBB9_2
; GFX942-NEXT: ; %bb.1: ; %bb.1
; GFX942-NEXT: global_load_dwordx2 v[2:3], v1, s[10:11]
+; GFX942-NEXT: s_mov_b32 s4, 0
; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 7, v0
+; GFX942-NEXT: s_mov_b32 s5, s4
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_mov_b32_e32 v4, 0
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX942-NEXT: s_andn2_b64 s[0:1], s[0:1], exec
; GFX942-NEXT: s_and_b64 s[4:5], vcc, exec
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
; GFX942-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
; GFX942-NEXT: .LBB9_2: ; %Flow
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
@@ -534,14 +535,14 @@ define amdgpu_kernel void @v8i8_phi_const(ptr addrspace(1) %src1, ptr addrspace(
; GFX942-NEXT: v_cmp_gt_u32_e32 vcc, 7, v16
; GFX942-NEXT: s_andn2_b64 s[0:1], s[0:1], exec
; GFX942-NEXT: s_and_b64 s[4:5], vcc, exec
-; GFX942-NEXT: v_mov_b32_e32 v4, 8
-; GFX942-NEXT: v_mov_b32_e32 v5, 7
-; GFX942-NEXT: v_mov_b32_e32 v6, 6
-; GFX942-NEXT: v_mov_b32_e32 v1, 5
-; GFX942-NEXT: v_mov_b32_e32 v7, 4
-; GFX942-NEXT: v_mov_b32_e32 v8, 3
-; GFX942-NEXT: v_mov_b32_e32 v9, 2
; GFX942-NEXT: v_mov_b32_e32 v0, 1
+; GFX942-NEXT: v_mov_b32_e32 v9, 2
+; GFX942-NEXT: v_mov_b32_e32 v8, 3
+; GFX942-NEXT: v_mov_b32_e32 v7, 4
+; GFX942-NEXT: v_mov_b32_e32 v1, 5
+; GFX942-NEXT: v_mov_b32_e32 v6, 6
+; GFX942-NEXT: v_mov_b32_e32 v5, 7
+; GFX942-NEXT: v_mov_b32_e32 v4, 8
; GFX942-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_lshrrev_b32_e32 v15, 24, v3
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.ll b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.ll
index d54731f15a89f..365b7fa5d02bc 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.ll
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.ll
@@ -14,12 +14,10 @@ define amdgpu_kernel void @ds_prefetch_flushed(ptr addrspace(3) %lds, ptr addrsp
; CHECK-NEXT: s_clause 0x1
; CHECK-NEXT: s_load_b32 s1, s[4:5], 0x0 nv
; CHECK-NEXT: s_load_b32 s0, s[4:5], 0x10 nv
-; CHECK-NEXT: s_mov_b32 s2, 0x3ff
-; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; CHECK-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_bitop2_b32 v10, s2, v0 bitop3:0x40
-; CHECK-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v6, v4
-; CHECK-NEXT: v_dual_mov_b32 v7, v4 :: v_dual_mov_b32 v8, v4
-; CHECK-NEXT: v_mov_b32_e32 v9, v4
+; CHECK-NEXT: v_and_b32_e32 v10, 0x3ff, v0
+; CHECK-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0
+; CHECK-NEXT: v_dual_mov_b32 v6, 0 :: v_dual_mov_b32 v7, 0
+; CHECK-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v9, 0
; CHECK-NEXT: s_wait_kmcnt 0x0
; CHECK-NEXT: v_lshl_add_u32 v11, v10, 6, s1
; CHECK-NEXT: v_lshl_add_u32 v12, v10, 5, s1
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll
index 810cae94cc418..5be39f55a811f 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll
@@ -14,11 +14,9 @@ define amdgpu_kernel void @ds_prefetch_pattern(ptr addrspace(3) %lds, ptr addrsp
; CHECK-NEXT: s_clause 0x1
; CHECK-NEXT: s_load_b32 s1, s[4:5], 0x0 nv
; CHECK-NEXT: s_load_b32 s0, s[4:5], 0x10 nv
-; CHECK-NEXT: s_mov_b32 s2, 0x3ff
-; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; CHECK-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_bitop2_b32 v12, s2, v0 bitop3:0x40
-; CHECK-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
-; CHECK-NEXT: v_mov_b32_e32 v3, v0
+; CHECK-NEXT: v_and_b32_e32 v12, 0x3ff, v0
+; CHECK-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; CHECK-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
; CHECK-NEXT: s_wait_kmcnt 0x0
; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_3)
; CHECK-NEXT: v_lshl_add_u32 v13, v12, 8, s1
diff --git a/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll b/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll
index 9989907f4b9b9..7fb97edf097d8 100644
--- a/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll
+++ b/llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll
@@ -176,7 +176,7 @@ define amdgpu_gfx void @strict_wwm_cfg(ptr addrspace(8) inreg %tmp14, i32 %arg)
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v1
; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[36:37], v0, s34
-; GFX9-O0-NEXT: v_mov_b32_e32 v0, s34
+; GFX9-O0-NEXT: v_mov_b32_e32 v0, 0
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 s[34:35], exec
; GFX9-O0-NEXT: v_writelane_b32 v5, s34, 4
diff --git a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
index 0ef5725607920..ad895ad4190dd 100644
--- a/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
+++ b/llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
@@ -153,7 +153,7 @@ define amdgpu_cs void @cfg(ptr addrspace(8) inreg %tmp14, i32 %arg) {
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v1
; GFX9-O0-NEXT: buffer_store_dword v3, off, s[16:19], 0 offset:8 ; 4-byte Folded Spill
; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[2:3], v0, s0
-; GFX9-O0-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-O0-NEXT: v_mov_b32_e32 v0, 0
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[16:19], 0 offset:4 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 s[0:1], exec
; GFX9-O0-NEXT: v_writelane_b32 v5, s0, 5
@@ -1016,7 +1016,7 @@ define amdgpu_cs void @strict_wwm_cfg(ptr addrspace(8) inreg %tmp14, i32 %arg) {
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v1
; GFX9-O0-NEXT: buffer_store_dword v3, off, s[16:19], 0 offset:8 ; 4-byte Folded Spill
; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[2:3], v0, s0
-; GFX9-O0-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-O0-NEXT: v_mov_b32_e32 v0, 0
; GFX9-O0-NEXT: buffer_store_dword v0, off, s[16:19], 0 offset:4 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 s[0:1], exec
; GFX9-O0-NEXT: v_writelane_b32 v5, s0, 5
More information about the llvm-commits
mailing list